Search Models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from elasticsearch import Elasticsearch\n",
    "import requests\n",
    "from tqdm.notebook import tqdm\n",
    "import re \n",
    "import numpy as np\n",
    "from numpy import dot\n",
    "from numpy.linalg import norm\n",
    "import hnswlib\n",
    "from collections import defaultdict, Counter\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "module_url = \"./module/UnivTrans\" #@param [\"https://tfhub.dev/google/universal-sentence-encoder/4\", \"https://tfhub.dev/google/universal-sentence-encoder-large/5\"]\n",
    "model = hub.load(module_url)\n",
    "def embed(inputText):\n",
    "    return model(inputText)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fasttext\n",
    "fasttextModel = fasttext.load_model('crawl-300d-2M-subword.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getQueryEmbedding(query, embeddingType='fastText'):\n",
    "    \n",
    "    '''Get embedding for a single query. Query is pre-processed in this function itself'''\n",
    "\n",
    "    translator = str.maketrans('', '', string.punctuation) \n",
    "    query.strip()\n",
    "    query = query.translate(translator)\n",
    "    query = ' '.join(query.split())\n",
    "    \n",
    "    if embeddingType == 'fastText':\n",
    "        embedding = fasttextModel.get_word_vector(query)\n",
    "    elif embeddingType == 'USE':\n",
    "        embedding = embed([query])[0].numpy()\n",
    "        \n",
    "    return np.asarray(embedding)\n",
    "\n",
    "\n",
    "def getQueryEmbeddings(queryList, embeddingType='fastText'):\n",
    "    '''Get embedding list for a list of queries. Query is pre-processed in this function itself''' \n",
    "    embeddings = []\n",
    "    \n",
    "    ## Preprocessing\n",
    "    translator = str.maketrans('', '', string.punctuation) \n",
    "    for i in range(len(queryList)):\n",
    "            queryList[i].strip()\n",
    "            queryList[i] = queryList[i].translate(translator)\n",
    "            queryList[i] = ' '.join(queryList[i].split())\n",
    "    \n",
    "    if embeddingType == 'fastText':\n",
    "        for query in queryList:\n",
    "            embedding = fasttextModel.get_word_vector(query)\n",
    "            embeddings.append(embedding)\n",
    "    elif embeddingType == 'USE':\n",
    "        embeddings = embed(queryList).numpy()\n",
    "        \n",
    "    return np.asarray(embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getDocumentEmbeddings(docIDList, method='abstract', embeddingType='fastText'):\n",
    "    docIDSet = set(docIDList)\n",
    "    embeddingDictForDocs = dict()\n",
    "    \n",
    "    if method=='abstract':\n",
    "        if embeddingType == 'fastText':\n",
    "            filename = './data/dblpAbstract_2Thresholded_FT_Embeddings.json'\n",
    "        elif embeddingType == 'USE':\n",
    "            filename = './data/dblp_Abstract_2Thresholded_USE_Trans_Embeddings.json'\n",
    "        elif embeddingType == 'tfIdf':\n",
    "            filename = './data/dblpAbstract_2Thresholded_TfIdfUni_Embeddings.json'\n",
    "        with open(filename, 'r') as file:\n",
    "                for line in file:\n",
    "                    data = json.loads(line)\n",
    "                    if data['id'] in docIDSet:\n",
    "                        embeddingDictForDocs[data['id']] = data['embedding']\n",
    "                        \n",
    "    elif method=='title':\n",
    "        if embeddingType == 'fastText':\n",
    "            filename = './data/dblpTitle_2Thresholded_FT_Embeddings.json'\n",
    "        elif embeddingType == 'USE':\n",
    "            filename = './data/dblp_Title_2Thresholded_USE_Trans_Embeddings.json'\n",
    "        with open(filename, 'r') as file:\n",
    "                for line in file:\n",
    "                    data = json.loads(line)\n",
    "                    if data['id'] in docIDSet:\n",
    "                        embeddingDictForDocs[data['id']] = data['embedding']\n",
    "    \n",
    "    elif method=='fos':\n",
    "        records = []\n",
    "        PapersOutFileName = './data/es/dblp_AIpapers_v1.json'\n",
    "        i = 0\n",
    "\n",
    "        with open(PapersOutFileName, 'r') as file:\n",
    "            for line in file:\n",
    "                if i % 2 != 0:\n",
    "                    data = json.loads(line)\n",
    "                    if data['id'] in docIDSet:\n",
    "                        records.append(data)\n",
    "                i += 1\n",
    "        \n",
    "        assert len(records) == len(docIDList)\n",
    "        \n",
    "        if embeddingType == 'fastText':\n",
    "            fileName = './data/dblp_fos_FT_Phrase_embeddings.json'\n",
    "        elif embeddingType == 'USE':\n",
    "            fileName = './data/dblp_fos_USE_embeddings.json'\n",
    "        \n",
    "        embeddingDict = dict()\n",
    "        with open(fileName, 'r') as file:\n",
    "            for line in file:\n",
    "                data = json.loads(line)\n",
    "                embeddingDict[data['fos']] = np.asarray(data['embedding']) \n",
    "        \n",
    "        for record in tqdm(records):\n",
    "            recordEmbeddingList = []\n",
    "            for fos in record['fos']:\n",
    "                recordEmbeddingList.append(embeddingDict[fos])\n",
    "            embeddingDictForDocs[record['id']] = np.mean(recordEmbeddingList, axis = 0)\n",
    "           \n",
    "            \n",
    "    elif method=='fosIdf':\n",
    "        records = []\n",
    "        PapersOutFileName = './data/es/dblp_AIpapers_v1.json'\n",
    "        i = 0\n",
    "\n",
    "        with open(PapersOutFileName, 'r') as file:\n",
    "            for line in file:\n",
    "                if i % 2 != 0:\n",
    "                    data = json.loads(line)\n",
    "                    records.append(data)\n",
    "                i += 1\n",
    "        \n",
    "        if embeddingType == 'fastText':\n",
    "            fileName = './data/dblp_fos_FT_Phrase_embeddings.json'\n",
    "        elif embeddingType == 'USE':\n",
    "            filename = './data/dblp_fos_USE_embeddings.json'\n",
    "        \n",
    "        embeddingDict = dict()\n",
    "        fosCount = dict()\n",
    "        N = len(records)\n",
    "        with open(filename, 'r') as file:\n",
    "            for line in file:\n",
    "                data = json.loads(line)\n",
    "                embeddingDict[data['fos']] = np.asarray(data['embedding']) \n",
    "                fosCount[data['fos']] = data['count']\n",
    "        \n",
    "        for record in tqdm(records):\n",
    "            recordEmbeddingList = []\n",
    "            weightList = []\n",
    "            for fos in record['fos']:\n",
    "                recordEmbeddingList.append(embeddingDict[fos] * (N / fosCount[fos]))\n",
    "                weightList.append((N / fosCount[fos]))\n",
    "            embeddingDictForDocs[record['id']] = np.mean(recordEmbeddingList, axis = 0) / np.sum(weightList)\n",
    "            \n",
    "    embeddings = []\n",
    "    for docID in docIDList:\n",
    "        embeddings.append(embeddingDictForDocs[docID])\n",
    "    assert len(embeddings) == len(docIDList)\n",
    "    return np.asarray(embeddings)\n",
    "\n",
    "\n",
    "\n",
    "def getAllDocumentEmbeddings(method='abstract', embeddingType='fastText'):\n",
    "    \n",
    "    embeddings = []\n",
    "    \n",
    "    if method=='abstract':\n",
    "        if embeddingType == 'fastText':\n",
    "            filename = './data/dblpAbstract_2Thresholded_FT_Embeddings.json'\n",
    "        elif embeddingType == 'USE':\n",
    "            filename = './data/dblp_Abstract_2Thresholded_USE_Trans_Embeddings.json'\n",
    "        with open(filename, 'r') as file:\n",
    "                for line in file:\n",
    "                    data = json.loads(line)\n",
    "                    embedding = data['embedding']\n",
    "                    embeddings.append(embedding)\n",
    "    elif method=='title':\n",
    "        if embeddingType == 'fastText':\n",
    "            filename = './data/dblpTitle_2Thresholded_FT_Embeddings.json'\n",
    "        elif embeddingType == 'USE':\n",
    "            filename = './data/dblp_Title_2Thresholded_USE_Trans_Embeddings.json'\n",
    "        with open(filename, 'r') as file:\n",
    "                for line in file:\n",
    "                    data = json.loads(line)\n",
    "                    embedding = data['embedding']\n",
    "                    embeddings.append(embedding)\n",
    "    \n",
    "    elif method=='fos':\n",
    "        records = []\n",
    "        PapersOutFileName = './data/es/dblp_AIpapers_v1.json'\n",
    "        i = 0\n",
    "\n",
    "        with open(PapersOutFileName, 'r') as file:\n",
    "            for line in file:\n",
    "                if i % 2 != 0:\n",
    "                    data = json.loads(line)\n",
    "                    records.append(data)\n",
    "                i += 1\n",
    "        \n",
    "        if embeddingType == 'fastText':\n",
    "            fileName = './data/dblp_fos_FT_Phrase_embeddings.json'\n",
    "        elif embeddingType == 'USE':\n",
    "            filename = './data/dblp_fos_USE_embeddings.json'\n",
    "        \n",
    "        embeddingDict = dict()\n",
    "        with open(fileName, 'r') as file:\n",
    "            for line in file:\n",
    "                data = json.loads(line)\n",
    "                embeddingDict[data['fos']] = np.asarray(data['embedding']) \n",
    "        \n",
    "        for record in tqdm(records):\n",
    "            recordEmbeddingList = []\n",
    "            for fos in record['fos']:\n",
    "                recordEmbeddingList.append(embeddingDict[fos])\n",
    "            embeddings.append(np.mean(recordEmbeddingList, axis = 0))\n",
    "            \n",
    "    elif method=='fosIdf':\n",
    "        records = []\n",
    "        PapersOutFileName = './data/es/dblp_AIpapers_v1.json'\n",
    "        i = 0\n",
    "\n",
    "        with open(PapersOutFileName, 'r') as file:\n",
    "            for line in file:\n",
    "                if i % 2 != 0:\n",
    "                    data = json.loads(line)\n",
    "                    records.append(data)\n",
    "                i += 1\n",
    "        \n",
    "        if embeddingType == 'fastText':\n",
    "            fileName = './data/dblp_fos_FT_Phrase_embeddings.json'\n",
    "        elif embeddingType == 'USE':\n",
    "            fileName = './data/dblp_fos_USE_embeddings.json'\n",
    "        \n",
    "        embeddingDict = dict()\n",
    "        fosCount = dict()\n",
    "        N = len(records)\n",
    "        with open(fileName, 'r') as file:\n",
    "            for line in file:\n",
    "                data = json.loads(line)\n",
    "                embeddingDict[data['fos']] = np.asarray(data['embedding']) \n",
    "                fosCount[data['fos']] = data['count']\n",
    "        \n",
    "        for record in tqdm(records):\n",
    "            recordEmbeddingList = []\n",
    "            weightList = []\n",
    "            for fos in record['fos']:\n",
    "                recordEmbeddingList.append(embeddingDict[fos] * (N / fosCount[fos]))\n",
    "                weightList.append((N / fosCount[fos]))\n",
    "            embeddings.append(np.mean(recordEmbeddingList, axis = 0) / np.sum(weightList))\n",
    "    \n",
    "    return np.asarray(embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Search using Direct embedding Match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def buildIndexer(docEmbeddings):\n",
    "    numElements = len(docEmbeddings)\n",
    "    dimension = len(docEmbeddings[0])\n",
    "    embeddings = np.asarray(docEmbeddings)\n",
    "    data_labels = np.arange(numElements)\n",
    "    p = hnswlib.Index(space = 'cosine', dim = dimension) # possible options are l2, cosine or ip\n",
    "    p.init_index(max_elements = numElements, ef_construction = 200, M = dimension)\n",
    "    p.add_items(embeddings, data_labels)\n",
    "    p.set_ef(30) \n",
    "    \n",
    "    return p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadIndexer(filepath, numElements):\n",
    "    p = hnswlib.Index(space='cosine', dim=dimension)  # the space can be changed - keeps the data, alters the distance function.\n",
    "    p.load_index(\"./models/fastTexthnswlib.bin\", max_elements =numElements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def searchWithEmbedding(queryList, K=10, method='abstract', embeddingType='fastText'):\n",
    "    IDList = []\n",
    "    with open('./data/dblp_AIpapers2Thresholded.json', 'r') as file:\n",
    "        for line in file:\n",
    "            data = json.loads(line)\n",
    "            IDList.append(data['id'])\n",
    "\n",
    "    docEmbeddings = getAllDocumentEmbeddings(method=method, embeddingType=embeddingType)\n",
    "    queryEmbeddings = getQueryEmbeddings(queryList, embeddingType=embeddingType)\n",
    "    p = buildIndexer(docEmbeddings)\n",
    "\n",
    "    labels, _ = p.knn_query(queryEmbeddings, k = K )\n",
    "    for i in range(len(labels)):\n",
    "        labels[i] = [IDList[ind] for ind in labels[i]]\n",
    "        \n",
    "    return labels\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from numpy import dot\n",
    "from numpy.linalg import norm\n",
    "\n",
    "def cosine_similarity(a, b):\n",
    "    cos_sim = dot(a, b)/(norm(a)*norm(b))\n",
    "    return cos_sim\n",
    "def weight(score, esScore):\n",
    "    embeddingWeight = 0.8\n",
    "    s = embeddingWeight * score + (1 - embeddingWeight) * esScore\n",
    "    return s\n",
    "\n",
    "\n",
    "def normalize(lis):\n",
    "    _min = min(lis)\n",
    "    _max = max(lis)\n",
    "    lis  = [(x - _min)/(_max - _min) for x in lis]\n",
    "    return lis\n",
    "\n",
    "def rankList(query, docList, esScoreList=None, K=10,  method ='fos', embeddingType='fastText'):\n",
    "    '''\n",
    "    ReRank documents in the docList wrt the query\n",
    "\n",
    "    Parameters: \n",
    "    query (str): query wrt. which the documents will be ranked\n",
    "\n",
    "    doclist(list[str]): IDs of the documents to rerank, len(doclist >= K)\n",
    "\n",
    "    esScoreList(list[int]): scores of the documents as returned by Elastic Search\n",
    "\n",
    "    k(int): number of top documents to return after re-ranking\n",
    "\n",
    "    Returns: \n",
    "    list[str]: re-ranked list of document IDs\n",
    "    '''\n",
    "    queryEmbedding = getQueryEmbedding(query, embeddingType=embeddingType)\n",
    "    docEmbeddings = getDocumentEmbeddings(docList, method = method, embeddingType=embeddingType)\n",
    "    docEmbeddings = docEmbeddings.tolist()\n",
    "    cosineSimScores = [ cosine_similarity(queryEmbedding, np.asarray(docEmbedding)) for docEmbedding in docEmbeddings]\n",
    "    if esScoreList is None:\n",
    "        IDsWithScore = [(score, ID) for score, ID in zip(cosineSimScores, docList)]\n",
    "    elif esScoreList is not None:\n",
    "         IDsWithScore = [(weight(score, esScore), ID) for score, ID, esScore in zip(cosineSimScores, docList, esScoreList)]\n",
    "    IDsWithScore.sort(reverse=True)\n",
    "    IDsWithScore = IDsWithScore[:K + 1]                    ## Keep top-K documents only\n",
    "    \n",
    "    return [ID for _,ID in IDsWithScore]\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Search with Elastic Search\n",
    "es = Elasticsearch([{'host': 'localhost', 'port': 9200}])\n",
    "\n",
    "def elasticSearch(queryList, index='dblp_v1', K = 10):\n",
    "    fields = ['id', 'title', 'venue', 'authors', 'year', 'abstract', 'fos']\n",
    "    queryBody = {\n",
    "    \"query\": {\n",
    "        \"multi_match\" : {\n",
    "            \"query\" : \"sentence embeddings\",\n",
    "            \"fields\" : ['title', 'abstract', 'authors']\n",
    "        }\n",
    "    }\n",
    "}\n",
    "    queryBody['size'] = K\n",
    "    searchResults = []\n",
    "    for query in queryList:\n",
    "        queryBody['query']['multi_match']['query'] = query\n",
    "        res= es.search(index=index,body=queryBody)\n",
    "        searchResults.append([hit['_id'] for hit in res['hits']['hits']])\n",
    "    return searchResults\n",
    "\n",
    "def rankedElasticSearch(queryList, index='dblp_v1', K = 10, includeEsScores=False, rerank=True):\n",
    "    fields = ['id', 'title', 'venue', 'authors', 'year', 'abstract', 'fos']\n",
    "    queryBody = {\n",
    "    \"query\": {\n",
    "        \"multi_match\" : {\n",
    "            \"query\" : \"sentence embeddings\",\n",
    "            \"fields\" : ['title^3', 'abstract', 'authors', 'fos^2']\n",
    "        }\n",
    "    }\n",
    "}\n",
    "    if rerank:\n",
    "        queryBody['size'] = 2 * K\n",
    "    else:\n",
    "        queryBody['size'] = K\n",
    "    queryBody['query']['multi_match']['fuzziness'] = 'AUTO'\n",
    "    searchResults = []\n",
    "    for query in queryList:\n",
    "        queryBody['query']['multi_match']['query'] = query\n",
    "        res= es.search(index=index,body=queryBody)\n",
    "        initList = [hit['_id'] for hit in res['hits']['hits']]\n",
    "        esScoreList = [hit['_score'] for hit in res['hits']['hits']]\n",
    "        esScoreList = normalize(esScoreList)\n",
    "        if rerank:\n",
    "            if includeEsScores:\n",
    "                searchResults.append(rankList(query, initList, esScoreList = esScoreList))\n",
    "            else:\n",
    "                searchResults.append(rankList(query, initList))\n",
    "        else:\n",
    "            searchResults.append(initList)\n",
    "    return searchResults\n",
    "\n",
    "def rankedMLTElasticSearch(queryList, index='dblp_v1', K = 10, includeEsScores=False, rerank = True):\n",
    "    fields = ['id', 'title', 'venue', 'authors', 'year', 'abstract', 'fos']\n",
    "    queryBody = {\n",
    "    \"query\": {\n",
    "        \"more_like_this\" : {\n",
    "            \"like\" : \"sentence embeddings\",\n",
    "            \"fields\" : ['title^3', 'abstract', 'authors', 'fos^2'],\n",
    "            \"min_term_freq\" : 1\n",
    "        }\n",
    "    }\n",
    "}    \n",
    "    queryBody['size'] = 5 * K\n",
    "    \n",
    "    searchResults = []\n",
    "    for query in queryList:\n",
    "        queryBody['query']['more_like_this']['like'] = query\n",
    "        res= es.search(index=index,body=queryBody)\n",
    "        initList = [hit['_id'] for hit in res['hits']['hits']]\n",
    "        esScoreList = [hit['_score'] for hit in res['hits']['hits']] \n",
    "        esScoreList = normalize(esScoreList)\n",
    "        if rerank:\n",
    "            if includeEsScores:\n",
    "                searchResults.append(rankList(query, initList, esScoreList = esScoreList))\n",
    "            else:\n",
    "                searchResults.append(rankList(query, initList))\n",
    "        else:\n",
    "            searchResults.append(initList)\n",
    "        \n",
    "        \n",
    "    return searchResults\n",
    "\n",
    "def MLT(docID, index='dblp_v1', K = 10):\n",
    "    fields = ['id', 'title', 'venue', 'authors', 'year', 'abstract', 'fos']\n",
    "    queryBody = {\n",
    "    \"query\": {\n",
    "        \"more_like_this\" : {\n",
    "            \"like\" : [\n",
    "            {\n",
    "                \"_index\" : index,\n",
    "                \"_id\" : docID\n",
    "            }\n",
    "            ],\n",
    "            \"fields\" : ['title', 'abstract', 'authors', 'fos'],\n",
    "            \"min_term_freq\" : 1\n",
    "        }\n",
    "    }\n",
    "}    \n",
    "    queryBody['size'] = K\n",
    "    res= es.search(index=index,body=queryBody)\n",
    "    return [hit['_id'] for hit in res['hits']['hits']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rankWithSVM(paperID, paperEmbedding, docList, embeddings, K = 10, method='abstract', embeddingType='USE'):\n",
    "    from sklearn import svm\n",
    "    import random\n",
    "    \n",
    "    with open(\"./data/orderedPaperIDs.json\", 'r') as f:\n",
    "        IDList = json.load(f)\n",
    "    negSampleSize = 2000\n",
    "    excludeSet = set(docList)\n",
    "    excludeSet.add(paperID)\n",
    "    negSamples = []\n",
    "    for id in IDList:\n",
    "        if id not in excludeSet:\n",
    "            negSamples.append(id)\n",
    "    random.shuffle(negSamples)\n",
    "    negSamples = negSamples[:negSampleSize]\n",
    "    \n",
    "    y = np.zeros(negSampleSize + 1, dtype=np.float32)\n",
    "    y[-1] = 1\n",
    "    trainEmbeddings =  getDocumentEmbeddings(negSamples , method=method, embeddingType=embeddingType).tolist()\n",
    "    trainEmbeddings.append(paperEmbedding)\n",
    "    trainEmbeddings = np.asarray(trainEmbeddings)\n",
    "    \n",
    "    c = list(zip(trainEmbeddings, y))\n",
    "    random.shuffle(c)\n",
    "    trainEmbeddings, y = zip(*c)\n",
    "    \n",
    "    clf = svm.LinearSVC(class_weight='balanced', verbose=True, tol=1e-4, C=0.1)\n",
    "    print('Fitting SVM')\n",
    "    clf.fit(trainEmbeddings, y)\n",
    "    print('SVM fitted')\n",
    "    s = clf.decision_function(np.asarray(embeddings))\n",
    "    ix = np.argsort(s)[:-K-1:-1]\n",
    "    return [docList[i] for i in ix]\n",
    "    \n",
    "    \n",
    "def rankListWithCosine(paperID, paperEmbedding, docList, embeddings, K = 10, esScoreList=None):\n",
    "    paperEmbedding = np.asarray(paperEmbedding)\n",
    "    \n",
    "    cosineSimScores = [ cosine_similarity(paperEmbedding, np.asarray(docEmbedding)) for docEmbedding in embeddings]\n",
    "    if esScoreList is None:\n",
    "        IDsWithScore = [(score, ID) for score, ID in zip(cosineSimScores, docList)]\n",
    "    elif esScoreList is not None:\n",
    "         IDsWithScore = [(weight(score, esScore), ID) for score, ID, esScore in zip(cosineSimScores, docList, esScoreList)]\n",
    "    IDsWithScore.sort(reverse=True)\n",
    "    IDsWithScore = IDsWithScore[:K + 1]                    ## Keep top-K documents only\n",
    "\n",
    "    return [ID for _,ID in IDsWithScore]\n",
    "    \n",
    "        \n",
    "\n",
    "def KClosestNodes(paperID, modelFile, K = 10, loadModel = None):\n",
    "    '''Given a paper, returns the K closest nodes(as per the augmented citation graph)'''\n",
    "    from gensim import models\n",
    "    if loadModel is None:\n",
    "        loadModel = models.keyedvectors.KeyedVectors.load_word2vec_format(modelFile)\n",
    "    \n",
    "    return [id for id, _ in loadModel.most_similar(paperID, topn=K)]\n",
    "\n",
    "def KClosestReranked(paperID, modelFile,K = 10, rerankScheme='exemplar', embeddingType='USE', method='abstract', loadModel = None):\n",
    "    '''Given a paper, reranks the closest nodes using text information of the papers'''\n",
    "    initDocList = KClosestNodes(paperID,  modelFile, K=3*K, loadModel = loadModel)\n",
    "    embeddings = getDocumentEmbeddings(initDocList + [paperID], method=method, embeddingType=embeddingType).tolist()\n",
    "    paperEmbedding = embeddings[-1]\n",
    "    embeddings.pop()\n",
    "    if rerankScheme == 'exemplar':\n",
    "        return rankWithSVM(paperID, paperEmbedding, initDocList, embeddings, K = 10, method=method, embeddingType=embeddingType)\n",
    "    elif rerankScheme == 'cosine':\n",
    "        return rankListWithCosine(paperID, paperEmbedding, initDocList, embeddings, K = 10, esScoreList=None)\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def esSeeder(queryList, index='dblp_v1', K = 10):\n",
    "#     fields = ['id', 'title', 'venue', 'authors', 'year', 'abstract', 'fos']\n",
    "#     queryBody = {\n",
    "#     \"query\": {\n",
    "#         \"multi_match\" : {\n",
    "#             \"query\" : \"sentence embeddings\",\n",
    "#             \"fields\" : ['title', 'abstract', 'authors', 'fos']\n",
    "#         }\n",
    "#     }\n",
    "# }\n",
    "#     queryBody['size'] = K\n",
    "#     searchResults = []\n",
    "#     esScores = []\n",
    "#     for query in queryList:\n",
    "#         queryBody['query']['multi_match']['query'] = query\n",
    "#         res= es.search(index=index,body=queryBody)\n",
    "#         searchResults.append([hit['_id'] for hit in res['hits']['hits']])\n",
    "#         esScoreList = [hit['_score'] for hit in res['hits']['hits']]\n",
    "#         esScoreList = normalize(esScoreList)\n",
    "#         esScores.append(esScoreList)\n",
    "#     return searchResults, esScores\n",
    "\n",
    "# def generateSeeds(queryList,K = K, method = method, embeddingType = embeddingType, index = index, seeder='es'):\n",
    "#     if seeder == 'es':\n",
    "#         return esSeeder(queryList = K, index = index, K = K)\n",
    "\n",
    "# def customMLT(queryList, K=10, method='title', embeddingType='USE', index='dblp_v1'):\n",
    "#     seedDocs, seedScores = generateSeeds(queryList,K=K, method=method, embeddingType = embeddingType, index = index, seeder='es')\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating Query Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "queryList = ['converting word to speech', 'Big data', 'efficient estimation of word representations in vector space', 'natural language interface', 'reinforcement learning in video game']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results1 = searchWithEmbedding(queryList, K=10, method='title', embeddingType='USE')\n",
    "results1 = [[str(ID) for ID in sublist] for sublist in results1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results2 = searchWithEmbedding(queryList, K=10, method='fos', embeddingType='fastText')\n",
    "results2 = [[str(ID) for ID in sublist] for sublist in results2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results3 = elasticSearch(queryList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "results4 = rankedElasticSearch(queryList, index='dblp_v1', K = 10, includeEsScores=False, rerank=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results5 = rankedMLTElasticSearch(queryList, index='dblp_v1', K = 10, includeEsScores=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resutlIDList = []\n",
    "for r1, r2, r3, r4, r5 in zip(results1, results2, results3, results4, results5):\n",
    "    lis = list(set().union(r1, r2, r3, r4, r5))\n",
    "    resutlIDList.append(lis)\n",
    "\n",
    "from functools import reduce\n",
    "import operator\n",
    "IDs = set(reduce(operator.concat, resutlIDList))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del model\n",
    "del fasttextModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PapersOutFileName = './data/es/dblp_AIpapers_v1.json'\n",
    "i = 0\n",
    "records = dict()\n",
    "with open(PapersOutFileName, 'r') as file:\n",
    "    for line in tqdm(file):\n",
    "        if i % 2 != 0:\n",
    "            data = json.loads(line)\n",
    "            if (data['id'] in IDs):\n",
    "                records[data['id']] = {'title': data['title'], 'abstract': data['abstract'], 'fos': ', '.join(data['fos'])}\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import random\n",
    "\n",
    "rows = []\n",
    "for query, idSubList in tqdm(zip(queryList, resutlIDList)):\n",
    "    for ID in idSubList:\n",
    "        localDict = records[ID]\n",
    "        rows.append([query, ID, localDict['title'], localDict['abstract'], localDict['fos'], 0])\n",
    "random.shuffle(rows)\n",
    "        \n",
    "with open('./data/annotations.csv', \"w\") as file:\n",
    "    writer = csv.writer(file)\n",
    "    header = ['query', 'id', 'title', 'abstract', 'fos', 'score']\n",
    "    writer.writerow(header)\n",
    "    for row in rows:\n",
    "            writer.writerow(row)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for lis in resutlIDList:\n",
    "    print(len(lis))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = [results1, results2, results3, results4, results5]\n",
    "len(results)\n",
    "\n",
    "with open('./data/search_results.json', 'w') as outfile:\n",
    "    for i in tqdm(range(len(results))):\n",
    "        outDict = dict()\n",
    "        outDict['id'] = i\n",
    "        outDict['result'] = results[i]\n",
    "        json.dump(outDict, outfile)\n",
    "        outfile.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ID in results6:\n",
    "        localDict = records[ID]\n",
    "        print(localDict['title'], localDict['abstract'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Evaluating Paper recommendation schemes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sequence: papers representive(in the same order) as queries in query list\n",
    "paperIDs = ['1583502834','2061556416', '1614298861', '168216068', '2123151547']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "results1 = []\n",
    "for paperID in paperIDs:\n",
    "    results1.append(MLT(paperID, index='dblp_v1', K = 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim import models\n",
    "modelFile = './models/node2vec_USE_Abstract_2Citation_Embeddings_WL_8_NN_42.kv'\n",
    "loadModel = models.keyedvectors.KeyedVectors.load_word2vec_format(modelFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "results2 = []\n",
    "for paperID in paperIDs:\n",
    "    results2.append(KClosestNodes(paperID, modelFile, K = 10, loadModel = loadModel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c47c786660e49fb94c884e9b2feb239",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "results3 = []\n",
    "for paperID in tqdm(paperIDs):\n",
    "    results3.append(KClosestReranked(paperID, modelFile,K = 10, rerankScheme='cosine', embeddingType='USE', method='abstract', loadModel = loadModel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa7d643d16884109b89978a4a98c72cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting SVM\n",
      "[LibLinear]SVM fitted\n",
      "Fitting SVM\n",
      "[LibLinear]SVM fitted\n",
      "Fitting SVM\n",
      "[LibLinear]SVM fitted\n",
      "Fitting SVM\n",
      "[LibLinear]SVM fitted\n",
      "Fitting SVM\n",
      "[LibLinear]SVM fitted\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results4 = []\n",
    "for paperID in tqdm(paperIDs):\n",
    "    results4.append(KClosestReranked(paperID, modelFile,K = 10, rerankScheme='exemplar', embeddingType='USE', method='abstract', loadModel = loadModel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63ab8f59950947b582a171ba799af2d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:5: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  \"\"\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "results5 = []\n",
    "for paperID in tqdm(paperIDs):\n",
    "    results5.append(KClosestReranked(paperID, modelFile,K = 10, rerankScheme='cosine', embeddingType='tfIdf', method='abstract', loadModel = loadModel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0b8bdfa088e4394ae67e7f2b6781b9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting SVM\n",
      "[LibLinear]SVM fitted\n",
      "Fitting SVM\n",
      "[LibLinear]SVM fitted\n",
      "Fitting SVM\n",
      "[LibLinear]SVM fitted\n",
      "Fitting SVM\n",
      "[LibLinear]SVM fitted\n",
      "Fitting SVM\n",
      "[LibLinear]SVM fitted\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results6 = []\n",
    "for paperID in tqdm(paperIDs):\n",
    "    results6.append(KClosestReranked(paperID, modelFile,K = 10, rerankScheme='exemplar', embeddingType='tfIdf', method='abstract',  loadModel = loadModel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "del loadModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "resutlIDList = []\n",
    "for r1, r2, r3, r4, r5, r6 in zip(results1, results2, results3, results4, results5, results6):\n",
    "    lis = list(set().union(r1, r2, r3, r4, r5, r6))\n",
    "    resutlIDList.append(lis)\n",
    "\n",
    "from functools import reduce\n",
    "import operator\n",
    "IDs = set(reduce(operator.concat, resutlIDList))\n",
    "IDs.update(paperIDs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "690ce5cf2ec94fb394bcc92ca9972b7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "PapersOutFileName = './data/es/dblp_AIpapers_v1.json'\n",
    "i = 0\n",
    "records = dict()\n",
    "with open(PapersOutFileName, 'r') as file:\n",
    "    for line in tqdm(file):\n",
    "        if i % 2 != 0:\n",
    "            data = json.loads(line)\n",
    "            if (data['id'] in IDs):\n",
    "                records[data['id']] = {'title': data['title'], 'abstract': data['abstract'], 'fos': ', '.join(data['fos'])}\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2790b17fa8b046c0bca52ce1cc457aa2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import random\n",
    "\n",
    "rows = []\n",
    "for paperID, idSubList in tqdm(zip(paperIDs, resutlIDList)):\n",
    "    for ID in idSubList:\n",
    "        localDict = records[ID]\n",
    "        rows.append([records[paperID]['title'], ID, localDict['title'], localDict['abstract'], localDict['fos'], 0])\n",
    "random.shuffle(rows)\n",
    "        \n",
    "with open('./data/recommendationAnnotations.csv', \"w\") as file:\n",
    "    writer = csv.writer(file)\n",
    "    header = ['paper', 'id', 'title', 'abstract', 'fos', 'score']\n",
    "    writer.writerow(header)\n",
    "    for row in rows:\n",
    "            writer.writerow(row)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8259e77aae2449658c7fefa52fbd3516",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=6.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "results = [results1, results2, results3, results4, results5, results6]\n",
    "len(results)\n",
    "\n",
    "with open('./data/recommendation_results.json', 'w') as outfile:\n",
    "    for i in tqdm(range(len(results))):\n",
    "        outDict = dict()\n",
    "        outDict['id'] = i\n",
    "        outDict['result'] = results[i]\n",
    "        json.dump(outDict, outfile)\n",
    "        outfile.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d84f8dc94014cecae87bc8ce35d74eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "titles = []\n",
    "IDList = []\n",
    "with open('./data/dblp_AIpapers2Thresholded.json', 'r') as file:\n",
    "    for line in tqdm(file):\n",
    "        data = json.loads(line)\n",
    "        titles.append(data['title'])\n",
    "        IDList.append(data['id'])\n",
    "def ret(paperID):\n",
    "    for id, title in zip(IDList, titles):\n",
    "        if (id == paperID):\n",
    "            return title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Novel-word pronunciation: a cross-language study',\n",
       "  'A pronunciation-by-analogy module for the Festival text-to-speech synthesiser',\n",
       "  'Improving pronunciation by analogy for text-to-speech applications.',\n",
       "  'Evaluating the pronunciation component of text-to-speech systems for English: a performance comparison of different approaches',\n",
       "  'A multistrategy approach to improving pronunciation by analogy',\n",
       "  'Can syllabification improve pronunciation by analogy of English',\n",
       "  'Text-to-speech conversion technology',\n",
       "  'Comparative evaluation of letter-to-sound conversion techniques for English text-to-speech synthesis',\n",
       "  'A Chinese text-to-speech system.',\n",
       "  'Word and syllable models for German text-to-speech synthesis.'],\n",
       " ['A Parallel Distributed Weka Framework for Big Data Mining Using Spark',\n",
       "  'Big Data Pre-processing: A Quality Framework',\n",
       "  'Evaluating the Quality of Social Media Data in Big Data Architecture',\n",
       "  'A Data Quality in Use model for Big Data',\n",
       "  'Data quality: The other face of Big Data',\n",
       "  'Processes Meet Big Data: Connecting Data Science with Process Science',\n",
       "  'Toward Scalable Systems for Big Data Analytics: A Technology Tutorial',\n",
       "  'Modern Enterprises in the Bubble: Why Big Data Matters',\n",
       "  'Research on Big Data Architecture, Key Technologies and Its Measures',\n",
       "  'Data mining with big data'],\n",
       " ['Distributed Representations of Words and Phrases and their Compositionality',\n",
       "  'Linguistic Regularities in Continuous Space Word Representations',\n",
       "  'Glove: Global Vectors for Word Representation',\n",
       "  'Distributed Representations of Sentences and Documents',\n",
       "  'A Structured Vector Space Model for Word Meaning in Context',\n",
       "  'Documents and Dependencies: an Exploration of Vector Space Models for Semantic Composition',\n",
       "  'Evaluating distributed word representations for capturing semantics of biomedical concepts',\n",
       "  'Modeling Order in Neural Word Embeddings at Scale',\n",
       "  'Efficient Non-parametric Estimation of Multiple Embeddings per Word in Vector Space',\n",
       "  'Measuring Word Significance using Distributed Representations of Words'],\n",
       " ['Natural Language Interface Construction Using Semantic Grammars',\n",
       "  'Eviza: A Natural Language Interface for Visual Analysis',\n",
       "  'Exploring evidence for shallow parsing',\n",
       "  'PANTO: A Portable Natural Language Interface to Ontologies',\n",
       "  'A Language Independent Shallow-Parser Compiler',\n",
       "  'The semantics-based natural language interface to relational databases',\n",
       "  'Robustness beyond shallowness: incremental deep parsing',\n",
       "  'Introduction to special issue on machine learning approaches to shallow parsing',\n",
       "  'Shallow Parsing Pipeline - Hindi-English Code-Mixed Social Media Text.',\n",
       "  'Prepositional Phrase Attachment in Shallow Parsing'],\n",
       " ['Learning to be a bot: reinforcement learning in shooter games',\n",
       "  'Creating a multi-purpose first person shooter bot with reinforcement learning',\n",
       "  'Hierarchical controller learning in a First-Person Shooter',\n",
       "  'DRE-Bot: A hierarchical First Person Shooter bot using multiple Sarsa(λ) reinforcement learners',\n",
       "  'Adaptive Shooting for Bots in First Person Shooter Games Using Reinforcement Learning',\n",
       "  'Learning policies for first person shooter games using inverse reinforcement learning',\n",
       "  'RETALIATE: learning winning policies in first-person shooter games',\n",
       "  'The Evolution of Gamebots for 3D First Person Shooter (FPS)',\n",
       "  'Neural networks training for weapon selection in first-person shooter games',\n",
       "  'Learning to intercept opponents in first person shooter games']]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[[ret(id) for id in sublist] for sublist in results1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['On the development of a name pronunciation system.',\n",
       "  'A Self-Learning Approach to Transcription of Danish Proper Names',\n",
       "  'Novel-word pronunciation: a cross-language study',\n",
       "  'Improved surname pronunciations using decision trees.',\n",
       "  'Phonetic transcription standards for european names (onomastica).',\n",
       "  'Pronouncing text by analogy',\n",
       "  'Learning Phonetic Rules in a Speech Recognition System',\n",
       "  'Proper name pronunciations for speech technology applications',\n",
       "  'Stochastic phonographic transduction for English',\n",
       "  'Predicting name pronunciation for a reverse directory service.'],\n",
       " ['Low latency analytics for streaming traffic data with Apache Spark',\n",
       "  'A Survey on Benchmarks for Big Data and Some More Considerations',\n",
       "  'Apache hadoop goes realtime at Facebook',\n",
       "  'CloudRank-D: benchmarking and ranking cloud computing systems for data processing applications',\n",
       "  'Hourglass: A library for incremental processing on Hadoop',\n",
       "  'Twitter Heron: Stream Processing at Scale',\n",
       "  \"Building LinkedIn's Real-time Activity Data Pipeline.\",\n",
       "  'Analyzing performance of Apache Tez and MapReduce with hadoop multinode cluster on Amazon cloud',\n",
       "  'A Brief Review on Leading Big Data Models',\n",
       "  'Thoth: towards managing a multi-system cluster'],\n",
       " ['Distributed Representations of Words and Phrases and their Compositionality',\n",
       "  'Glove: Global Vectors for Word Representation',\n",
       "  'Distributed Representations of Sentences and Documents',\n",
       "  'Hierarchical Attention Networks for Document Classification',\n",
       "  'Semantic Wide and Deep Learning for Detecting Crisis-Information Categories on Social Media',\n",
       "  'Convolutional Neural Networks for Sentence Classification',\n",
       "  'Joint Embedding of Hierarchical Categories and Entities for Concept Categorization and Dataless Classification.',\n",
       "  'Man is to computer programmer as woman is to homemaker? debiasing word embeddings',\n",
       "  'Deeper Attention to Abusive User Content Moderation',\n",
       "  'Compositional Recurrent Neural Networks for Chinese Short Text Classification'],\n",
       " ['Innovations in text interpretation',\n",
       "  'INTEGRATING TOP-DOWN AND BOTTOM-UP STRATEGIES IN A TEXT PROCESSING SYSTEM',\n",
       "  'INTERACTIVE SEMANTIC ANALYSIS OF TECHNICAL TEXTS',\n",
       "  'Processing dictionary definitions with phrasal pattern hierarchies',\n",
       "  'Natural Language Dialogue Service for Appointment Scheduling Agents',\n",
       "  'Exploiting aspectual features and connecting words for summarization-inspired temporal-relation extraction',\n",
       "  'Exploiting lexical regularities in designing natural language systems',\n",
       "  'Anaphoria in natural Language Understanding: A Survey',\n",
       "  'A logic programming based approach to QA@CLEF05 track',\n",
       "  'Understanding messages in a diagnostic domain'],\n",
       " ['Neuroevolutionary reinforcement learning for generalized helicopter control',\n",
       "  'Learning to play using low-complexity rule-based policies: illustrations through Ms. Pac-Man',\n",
       "  'Empirical Studies in Action Selection with Reinforcement Learning',\n",
       "  'Evaluating Human-like Behaviors of Video-Game Agents Autonomously Acquired with Biological Constraints',\n",
       "  'Applying reinforcement learning to small scale combat in the real-time strategy game StarCraft:Broodwar',\n",
       "  'Human-assisted neuroevolution through shaping, advice and examples',\n",
       "  'Creating Autonomous Adaptive Agents in a Real-Time First-Person Shooter Computer Game',\n",
       "  'An empirical analysis of value function-based and policy search reinforcement learning',\n",
       "  'Bootstrapping $Q$ -Learning for Robotics From Neuro-Evolution Results',\n",
       "  'Adaptive game AI with dynamic scripting']]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[[ret(id) for id in sublist] for sublist in results2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Improving pronunciation by analogy for text-to-speech applications.',\n",
       "  'Novel-word pronunciation: a cross-language study',\n",
       "  'A bi-directional model of English pronunciation.',\n",
       "  'On the development of a name pronunciation system.',\n",
       "  'Proper name pronunciations for speech technology applications',\n",
       "  'Multi-Lingual Testing of a Self-Learning Approach to Phonemic Transcription of Orthography',\n",
       "  'Predicting name pronunciation for a reverse directory service.',\n",
       "  'Phonetic transcription standards for european names (onomastica).',\n",
       "  'Segmental duration modelling in a text-to-speech system for the galician language.',\n",
       "  'Pmtools : A pronunciation modeling toolkit.',\n",
       "  'The pronunciation of unfamiliar native and non-native town names.'],\n",
       " ['Big data as the new enabler in business and other intelligence',\n",
       "  'Survey of real-time processing systems for big data',\n",
       "  'A Brief Review on Leading Big Data Models',\n",
       "  'SeqPig: simple and scalable scripting for large sequencing data sets in Hadoop',\n",
       "  'TPC-H Benchmark Analytics Scenarios and Performances on Hadoop Data Clouds',\n",
       "  'Big Data Generation',\n",
       "  'Big data system development: an embedded case study with a global outsourcing firm',\n",
       "  'Vision Paper: Towards an Understanding of the Limits of Map-Reduce Computation',\n",
       "  'Strategic Alignment of Cloud-Based Architectures for Big Data',\n",
       "  'A characterization of big data benchmarks',\n",
       "  'Hadoop at home: large-scale computing at a small college'],\n",
       " ['Glove: Global Vectors for Word Representation',\n",
       "  'Distributed Representations of Words and Phrases and their Compositionality',\n",
       "  'Improving Word Representations via Global Context and Multiple Word Prototypes',\n",
       "  'Distributed Representations of Sentences and Documents',\n",
       "  'Linguistic Regularities in Continuous Space Word Representations',\n",
       "  'Dynamic Pooling and Unfolding Recursive Autoencoders for Paraphrase Detection',\n",
       "  'Effective Use of Word Order for Text Categorization with Convolutional Neural Networks',\n",
       "  'A Convolutional Neural Network for Modelling Sentences',\n",
       "  'A Scalable Hierarchical Distributed Language Model',\n",
       "  'Convolutional Neural Networks for Sentence Classification',\n",
       "  'Compositional Recurrent Neural Networks for Chinese Short Text Classification'],\n",
       " ['Exploiting lexical regularities in designing natural language systems',\n",
       "  'A sublanguage approach to natural language processing for an expert system',\n",
       "  'Solving grammatical ambiguities within a surface syntactical parser for automatic indexing',\n",
       "  'A logic programming based approach to QA@CLEF05 track',\n",
       "  'Large lexicons for natural language processing: utilising the grammar coding system of LDOCE',\n",
       "  'Hybrid Semantic Analysis',\n",
       "  'COMPUTER AIDED INTERPRETATION OF LEXICAL COOCCURRENCES',\n",
       "  'An Integrated Archictecture for Shallow and Deep Processing',\n",
       "  'INTEGRATING TOP-DOWN AND BOTTOM-UP STRATEGIES IN A TEXT PROCESSING SYSTEM',\n",
       "  'Innovations in text interpretation',\n",
       "  'The linguistic string parser'],\n",
       " ['Creating Autonomous Adaptive Agents in a Real-Time First-Person Shooter Computer Game',\n",
       "  'Applying reinforcement learning to small scale combat in the real-time strategy game StarCraft:Broodwar',\n",
       "  'Adaptive game AI with dynamic scripting',\n",
       "  'Machine learning in digital games: a survey',\n",
       "  'Evaluating Human-like Behaviors of Video-Game Agents Autonomously Acquired with Biological Constraints',\n",
       "  'General video game playing',\n",
       "  'Open Loop Search for General Video Game Playing',\n",
       "  'Computer Go: A Grand Challenge to AI',\n",
       "  'Evolution and learning in an intrinsically motivated reinforcement learning robot',\n",
       "  'Learning to play using low-complexity rule-based policies: illustrations through Ms. Pac-Man',\n",
       "  'Bootstrapping $Q$ -Learning for Robotics From Neuro-Evolution Results']]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[[ret(id) for id in sublist] for sublist in results3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['On the development of a name pronunciation system.',\n",
       "  'Improving pronunciation by analogy for text-to-speech applications.',\n",
       "  'Predicting name pronunciation for a reverse directory service.',\n",
       "  'A bi-directional model of English pronunciation.',\n",
       "  'Proper name pronunciations for speech technology applications',\n",
       "  'The pronunciation of unfamiliar native and non-native town names.',\n",
       "  'Phonetic transcription standards for european names (onomastica).',\n",
       "  'Novel-word pronunciation: a cross-language study',\n",
       "  'Improved surname pronunciations using decision trees.',\n",
       "  'Segmental duration modelling in a text-to-speech system for the galician language.'],\n",
       " ['Big data as the new enabler in business and other intelligence',\n",
       "  'Hadoop at home: large-scale computing at a small college',\n",
       "  'Apache hadoop goes realtime at Facebook',\n",
       "  'SeqPig: simple and scalable scripting for large sequencing data sets in Hadoop',\n",
       "  'Big Data Generation',\n",
       "  'TPC-H Benchmark Analytics Scenarios and Performances on Hadoop Data Clouds',\n",
       "  'Survey of real-time processing systems for big data',\n",
       "  'Hourglass: A library for incremental processing on Hadoop',\n",
       "  'Analyzing performance of Apache Tez and MapReduce with hadoop multinode cluster on Amazon cloud',\n",
       "  'Clouds for Scalable Big Data Analytics'],\n",
       " ['Glove: Global Vectors for Word Representation',\n",
       "  'Improving Word Representations via Global Context and Multiple Word Prototypes',\n",
       "  'Linguistic Regularities in Continuous Space Word Representations',\n",
       "  'Distributed Representations of Words and Phrases and their Compositionality',\n",
       "  'Distributed Representations of Sentences and Documents',\n",
       "  'Effective Use of Word Order for Text Categorization with Convolutional Neural Networks',\n",
       "  '#TagSpace: Semantic Embeddings from Hashtags',\n",
       "  'Dynamic Pooling and Unfolding Recursive Autoencoders for Paraphrase Detection',\n",
       "  'Convolutional Neural Networks for Sentence Classification',\n",
       "  'Quantifying and Reducing Stereotypes in Word Embeddings.'],\n",
       " ['Exploiting lexical regularities in designing natural language systems',\n",
       "  'The linguistic string parser',\n",
       "  'A logic programming based approach to QA@CLEF05 track',\n",
       "  'Solving grammatical ambiguities within a surface syntactical parser for automatic indexing',\n",
       "  'Practical approach to knowledge-based question answering with natural language understanding and advanced reasoning',\n",
       "  'A definition and short history of Language Engineering',\n",
       "  'A sublanguage approach to natural language processing for an expert system',\n",
       "  'Large lexicons for natural language processing: utilising the grammar coding system of LDOCE',\n",
       "  'COMPUTER AIDED INTERPRETATION OF LEXICAL COOCCURRENCES',\n",
       "  'An Integrated Archictecture for Shallow and Deep Processing'],\n",
       " ['Creating Autonomous Adaptive Agents in a Real-Time First-Person Shooter Computer Game',\n",
       "  'Applying reinforcement learning to small scale combat in the real-time strategy game StarCraft:Broodwar',\n",
       "  'Adaptive game AI with dynamic scripting',\n",
       "  'Machine learning in digital games: a survey',\n",
       "  'Evaluating Human-like Behaviors of Video-Game Agents Autonomously Acquired with Biological Constraints',\n",
       "  'General video game playing',\n",
       "  'Open Loop Search for General Video Game Playing',\n",
       "  'Learning to play using low-complexity rule-based policies: illustrations through Ms. Pac-Man',\n",
       "  'Evolution and learning in an intrinsically motivated reinforcement learning robot',\n",
       "  'Computer Go: A Grand Challenge to AI']]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[[ret(id) for id in sublist] for sublist in results4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['On the development of a name pronunciation system.',\n",
       "  'A Self-Learning Approach to Transcription of Danish Proper Names',\n",
       "  'Novel-word pronunciation: a cross-language study',\n",
       "  'Improved surname pronunciations using decision trees.',\n",
       "  'Phonetic transcription standards for european names (onomastica).',\n",
       "  'Pronouncing text by analogy',\n",
       "  'Learning Phonetic Rules in a Speech Recognition System',\n",
       "  'Proper name pronunciations for speech technology applications',\n",
       "  'Stochastic phonographic transduction for English',\n",
       "  'Predicting name pronunciation for a reverse directory service.',\n",
       "  'Pmtools : A pronunciation modeling toolkit.'],\n",
       " ['A Brief Review on Leading Big Data Models',\n",
       "  'Big Data Generation',\n",
       "  'Big data as the new enabler in business and other intelligence',\n",
       "  'Strategic Alignment of Cloud-Based Architectures for Big Data',\n",
       "  'A fast and high throughput SQL query system for big data',\n",
       "  'BDGS: A Scalable Big Data Generator Suite in Big Data Benchmarking',\n",
       "  'Thoth: towards managing a multi-system cluster',\n",
       "  'Setting the Direction for Big Data Benchmark Standards',\n",
       "  'CloudRank-D: benchmarking and ranking cloud computing systems for data processing applications',\n",
       "  'A Survey on Benchmarks for Big Data and Some More Considerations',\n",
       "  'A Tale of Two Data-Intensive Paradigms: Applications, Abstractions, and Architectures'],\n",
       " ['Linguistic Regularities in Continuous Space Word Representations',\n",
       "  'Distributed Representations of Words and Phrases and their Compositionality',\n",
       "  'Dynamic Pooling and Unfolding Recursive Autoencoders for Paraphrase Detection',\n",
       "  'Glove: Global Vectors for Word Representation',\n",
       "  'Improving Word Representations via Global Context and Multiple Word Prototypes',\n",
       "  'Semantic Compositionality through Recursive Matrix-Vector Spaces',\n",
       "  'Distributed Representations of Sentences and Documents',\n",
       "  'Compositional Recurrent Neural Networks for Chinese Short Text Classification',\n",
       "  'Recurrent convolutional neural networks for text classification',\n",
       "  'Convolutional Neural Networks for Sentence Classification',\n",
       "  'Effective Use of Word Order for Text Categorization with Convolutional Neural Networks'],\n",
       " ['Exploiting lexical regularities in designing natural language systems',\n",
       "  'Large lexicons for natural language processing: utilising the grammar coding system of LDOCE',\n",
       "  'INTEGRATING TOP-DOWN AND BOTTOM-UP STRATEGIES IN A TEXT PROCESSING SYSTEM',\n",
       "  'A sublanguage approach to natural language processing for an expert system',\n",
       "  'The linguistic string parser',\n",
       "  'Practical approach to knowledge-based question answering with natural language understanding and advanced reasoning',\n",
       "  'Centered logic: the role of entity centered sentence representation in natural language inferencing',\n",
       "  'TRUMP : a transportable language understanding program',\n",
       "  'Understanding messages in a diagnostic domain',\n",
       "  'Transporting the linguistic string project system from a medical to a Navy domain',\n",
       "  'Anaphoria in natural Language Understanding: A Survey'],\n",
       " ['Applying reinforcement learning to small scale combat in the real-time strategy game StarCraft:Broodwar',\n",
       "  'Neuroevolutionary reinforcement learning for generalized helicopter control',\n",
       "  'Creating Autonomous Adaptive Agents in a Real-Time First-Person Shooter Computer Game',\n",
       "  'Machine learning in digital games: a survey',\n",
       "  'Evolution and learning in an intrinsically motivated reinforcement learning robot',\n",
       "  'Empirical Studies in Action Selection with Reinforcement Learning',\n",
       "  'Acceleration of reinforcement learning by a mobile robot using generalized rules',\n",
       "  'Distributed reinforcement learning for a traffic engineering application',\n",
       "  'General video game playing',\n",
       "  'Evaluating Human-like Behaviors of Video-Game Agents Autonomously Acquired with Biological Constraints',\n",
       "  'Evolution of reward functions for reinforcement learning']]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[[ret(id) for id in sublist] for sublist in results5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['The pronunciation of unfamiliar native and non-native town names.',\n",
       "  'Assigning phrase breaks from part-of-speech sequences.',\n",
       "  'Predicting name pronunciation for a reverse directory service.',\n",
       "  'Improving pronunciation by analogy for text-to-speech applications.',\n",
       "  'Automating the design of compact linguistic corpora.',\n",
       "  'A Self-Learning Approach to Transcription of Danish Proper Names',\n",
       "  'Phonetic transcription standards for european names (onomastica).',\n",
       "  'On the development of a name pronunciation system.',\n",
       "  'Multi-Lingual Testing of a Self-Learning Approach to Phonemic Transcription of Orthography',\n",
       "  'Proper name pronunciations for speech technology applications'],\n",
       " ['A Brief Review on Leading Big Data Models',\n",
       "  'Big Data Generation',\n",
       "  'Strategic Alignment of Cloud-Based Architectures for Big Data',\n",
       "  'Big data as the new enabler in business and other intelligence',\n",
       "  'A fast and high throughput SQL query system for big data',\n",
       "  'BDGS: A Scalable Big Data Generator Suite in Big Data Benchmarking',\n",
       "  'Thoth: towards managing a multi-system cluster',\n",
       "  'A Survey on Benchmarks for Big Data and Some More Considerations',\n",
       "  'Setting the Direction for Big Data Benchmark Standards',\n",
       "  'CloudRank-D: benchmarking and ranking cloud computing systems for data processing applications'],\n",
       " ['Linguistic Regularities in Continuous Space Word Representations',\n",
       "  'Distributed Representations of Words and Phrases and their Compositionality',\n",
       "  'Dynamic Pooling and Unfolding Recursive Autoencoders for Paraphrase Detection',\n",
       "  'Glove: Global Vectors for Word Representation',\n",
       "  'Semantic Compositionality through Recursive Matrix-Vector Spaces',\n",
       "  'Improving Word Representations via Global Context and Multiple Word Prototypes',\n",
       "  'Distributed Representations of Sentences and Documents',\n",
       "  'Convolutional Neural Networks for Sentence Classification',\n",
       "  'Poincaré Embeddings for Learning Hierarchical Representations',\n",
       "  'Hierarchical Attention Networks for Document Classification'],\n",
       " ['Exploiting lexical regularities in designing natural language systems',\n",
       "  'INTEGRATING TOP-DOWN AND BOTTOM-UP STRATEGIES IN A TEXT PROCESSING SYSTEM',\n",
       "  'Large lexicons for natural language processing: utilising the grammar coding system of LDOCE',\n",
       "  'A sublanguage approach to natural language processing for an expert system',\n",
       "  'The linguistic string parser',\n",
       "  'Centered logic: the role of entity centered sentence representation in natural language inferencing',\n",
       "  'Understanding messages in a diagnostic domain',\n",
       "  'Practical approach to knowledge-based question answering with natural language understanding and advanced reasoning',\n",
       "  'TRUMP : a transportable language understanding program',\n",
       "  'Anaphoria in natural Language Understanding: A Survey'],\n",
       " ['Applying reinforcement learning to small scale combat in the real-time strategy game StarCraft:Broodwar',\n",
       "  'Neuroevolutionary reinforcement learning for generalized helicopter control',\n",
       "  'Creating Autonomous Adaptive Agents in a Real-Time First-Person Shooter Computer Game',\n",
       "  'Machine learning in digital games: a survey',\n",
       "  'Distributed reinforcement learning for a traffic engineering application',\n",
       "  'Acceleration of reinforcement learning by a mobile robot using generalized rules',\n",
       "  'Evaluating Human-like Behaviors of Video-Game Agents Autonomously Acquired with Biological Constraints',\n",
       "  'Evolution and learning in an intrinsically motivated reinforcement learning robot',\n",
       "  'Adaptive game AI with dynamic scripting',\n",
       "  'Evolution of reward functions for reinforcement learning']]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[[ret(id) for id in sublist] for sublist in results6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "149"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}