Andy-acquire-notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "a06bc3ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "from requests import get\n",
    "from bs4 import BeautifulSoup\n",
    "import os\n",
    "import json\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "60ec6495",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://github.com/search?p=1&q=covid-19&type=Repositories'\n",
    "#headers = {'User-Agent': 'Codeup Data Science'}\n",
    "response = get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a6a077d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(response.content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "744c8f40",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CSSEGISandData/COVID-19'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all('a',class_='v-align-middle')[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "80cd7188",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CSSEGISandData/COVID-19'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all('a',class_='v-align-middle')[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "163bed18",
   "metadata": {},
   "outputs": [],
   "source": [
    "REPOS=[]\n",
    "for n in range(0,10):\n",
    "    ele= soup.find_all('a',class_='v-align-middle')\n",
    "    REPOS.append(ele[n].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "8dfc22d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CSSEGISandData/COVID-19',\n",
       " 'pcm-dpc/COVID-19',\n",
       " 'covid19india/covid19india-react',\n",
       " 'tokyo-metropolitan-gov/covid19',\n",
       " 'WorldHealthOrganization/app',\n",
       " 'nytimes/covid-19-data',\n",
       " 'owid/covid-19-data',\n",
       " 'MinCiencia/Datos-COVID19',\n",
       " 'someshkar/covid19india-cluster',\n",
       " 'pomber/covid19']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "REPOS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "6c8427a5",
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Input \u001b[0;32mIn [56]\u001b[0m, in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m soup \u001b[38;5;241m=\u001b[39m BeautifulSoup(response\u001b[38;5;241m.\u001b[39mcontent, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhtml.parser\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      7\u001b[0m ele\u001b[38;5;241m=\u001b[39m soup\u001b[38;5;241m.\u001b[39mfind_all(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ma\u001b[39m\u001b[38;5;124m'\u001b[39m,class_\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mv-align-middle\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 8\u001b[0m REPOS\u001b[38;5;241m.\u001b[39mappend(\u001b[43mele\u001b[49m\u001b[43m[\u001b[49m\u001b[43mn\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241m.\u001b[39mtext)\n\u001b[1;32m      9\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m10\u001b[39m)\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "REPOS=[]\n",
    "for i in range (1,11):\n",
    "    for n in range (0,10):\n",
    "        url = f'https://github.com/search?p={i}&q=covid-19&type=Repositories'\n",
    "        response = get(url)\n",
    "        soup = BeautifulSoup(response.content, 'html.parser')\n",
    "        ele= soup.find_all('a',class_='v-align-middle')\n",
    "        REPOS.append(ele[n].text)\n",
    "        time.sleep(10)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "1b8787cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "REPOS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "4f31205f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello\n",
      "world\n"
     ]
    }
   ],
   "source": [
    "print('hello')\n",
    "time.sleep(5)\n",
    "print('world')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "75f096a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "REPOS=pd.read_csv('repos.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "9b50f7dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "REPOS=REPOS['repo_names']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "88bd6bed",
   "metadata": {},
   "outputs": [],
   "source": [
    "REPOS=REPOS.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "4aa5a825",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CSSEGISandData/COVID-19',\n",
       " 'pcm-dpc/COVID-19',\n",
       " 'covid19india/covid19india-react',\n",
       " 'tokyo-metropolitan-gov/covid19',\n",
       " 'WorldHealthOrganization/app',\n",
       " 'nytimes/covid-19-data',\n",
       " 'owid/covid-19-data',\n",
       " 'MinCiencia/Datos-COVID19',\n",
       " 'someshkar/covid19india-cluster',\n",
       " 'pomber/covid19',\n",
       " 'MoH-Malaysia/covid19-public',\n",
       " 'github/covid19-dashboard',\n",
       " 'UCSD-AI4H/COVID-CT',\n",
       " 'neherlab/covid19_scenarios',\n",
       " 'datasets/covid-19',\n",
       " 'BlankerL/DXY-COVID-19-Data',\n",
       " 'k-sys/covid-19',\n",
       " 'kaz-ogiwara/covid19',\n",
       " 'ImperialCollegeLondon/covid19model',\n",
       " 'reichlab/covid19-forecast-hub',\n",
       " 'BlankerL/DXY-COVID-19-Crawler',\n",
       " 'cloudboxacademy/covid19',\n",
       " 'abuanwar072/Covid-19-Flutter-UI',\n",
       " 'italia/covid19-opendata-vaccini',\n",
       " 'FoldingAtHome/coronavirus',\n",
       " 'dssg-pt/covid19pt-data',\n",
       " 'thepanacealab/covid19_twitter',\n",
       " 'mathdroid/covid-19-api',\n",
       " 'dsfsi/covid19za',\n",
       " 'google/exposure-notifications-server',\n",
       " 'midas-network/COVID-19',\n",
       " 'echen102/COVID-19-TweetIDs',\n",
       " 'turicas/covid19-br',\n",
       " 'wcota/covid19br',\n",
       " 'OxCGRT/covid-policy-tracker',\n",
       " 'stevenliuyi/covid19',\n",
       " 'ccodwg/Covid19Canada',\n",
       " 'rizmaulana/kotlin-mvvm-covid19',\n",
       " 'openZH/covid_19',\n",
       " 'BrianRuizy/covid19-dashboard',\n",
       " 'JohnCoene/coronavirus',\n",
       " 'lispc/covid19-citymap-china',\n",
       " 'covid19datahub/COVID19',\n",
       " 'CSSEGISandData/COVID-19_Unified-Dataset',\n",
       " 'CodeForPhilly/chime',\n",
       " 'yuban01652/ucas-covid19',\n",
       " 'github/covid-19-repo-data',\n",
       " 'datameet/covid19',\n",
       " 'youyanggu/covid19_projections',\n",
       " 'reustle/covid19japan',\n",
       " 'LAB-MI/deplacement-covid-19',\n",
       " 'kavyanshpandey/Covid19--Tracker',\n",
       " 'Tencent/TH_COVID19_International',\n",
       " 'alsnhll/SEIR_COVID19',\n",
       " 'amodm/api-covid19-in',\n",
       " 'govex/COVID-19',\n",
       " 'anshumanpattnaik/covid19-full-stack-application',\n",
       " 'HzFu/COVID19_imaging_AI_paper_list',\n",
       " 'Lewuathe/COVID19-SIR',\n",
       " 'ncase/covid-19',\n",
       " 'Yu-Group/covid19-severity-prediction',\n",
       " 'daenuprobst/covid19-cases-switzerland',\n",
       " 'google/exposure-notifications-verification-server',\n",
       " 'disease-sh/API',\n",
       " 'Ank-Cha/Social-Distancing-Analyser-COVID-19',\n",
       " 'nihp-public/COVID-19-app-Android-BETA',\n",
       " 'TheEconomist/covid-19-excess-deaths-tracker',\n",
       " 'GoogleCloudPlatform/covid-19-open-data',\n",
       " 'ActiveConclusion/COVID19_mobility',\n",
       " 'wobsoriano/covid3d',\n",
       " 'SJang1/2021-korea-covid-19-remaining-vaccine-macro',\n",
       " 'Unicorndy/covid19_dashboard',\n",
       " 'twiecki/covid19',\n",
       " 'phildini/stayinghomeclub',\n",
       " 'globalcitizen/2019-wuhan-coronavirus-data',\n",
       " 'greenelab/covid19-review',\n",
       " 'covid19-forecast-hub-europe/covid19-forecast-hub-europe',\n",
       " 'minhealthnz/nz-covid-data',\n",
       " 'closedloop-ai/cv19index',\n",
       " 'cwoomi/cert-covid19',\n",
       " 'M3IT/COVID-19_Data',\n",
       " 'jannisborn/covid19_ultrasound',\n",
       " 'tomwhite/covid-19-uk-data',\n",
       " 'elcronos/COVID-19',\n",
       " 'muhammedtalo/COVID-19',\n",
       " 'trekhleb/covid-19',\n",
       " 'tirthajyoti/Covid-19-analysis',\n",
       " 'devarthurribeiro/covid19-brazil-api',\n",
       " 'dswh/voila-covid-19-dashboard',\n",
       " 'dungnb1333/SIIM-COVID19-Detection']"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "REPOS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "e58feb43",
   "metadata": {},
   "outputs": [],
   "source": [
    "bill=['Unicorndy/covid19_dashboard',\n",
    " 'twiecki/covid19',\n",
    " 'phildini/stayinghomeclub',\n",
    " 'globalcitizen/2019-wuhan-coronavirus-data',\n",
    " 'greenelab/covid19-review',\n",
    " 'covid19-forecast-hub-europe/covid19-forecast-hub-europe',\n",
    " 'minhealthnz/nz-covid-data',\n",
    " 'closedloop-ai/cv19index',\n",
    " 'jannisborn/covid19_ultrasound',\n",
    " 'tomwhite/covid-19-uk-data',\n",
    " 'elcronos/COVID-19',\n",
    " 'muhammedtalo/COVID-19',\n",
    " 'trekhleb/covid-19',\n",
    " 'tirthajyoti/Covid-19-analysis',\n",
    " 'devarthurribeiro/covid19-brazil-api',\n",
    " 'dswh/voila-covid-19-dashboard',\n",
    " 'nihp-public/COVID-19-app-iOS-BETA',\n",
    " 'chschoenenberger/covid19_dashboard',\n",
    " 'midudev/covid-19-spread-simulator',\n",
    " 'BDI-pathogens/OpenABM-Covid19',\n",
    " 'wareneutron/covid19-tracker-cli',\n",
    " 'hostolab/covidliste',\n",
    " 'imtsuki/bupt-ncov-report-action',\n",
    " 'heremaps/here-covid-19-tracker',\n",
    " 'Institut-Zdravotnych-Analyz/covid19-data',\n",
    " 'microsoft/Bing-COVID-19-Data',\n",
    " 'Priesemann-Group/covid19_inference_forecast',\n",
    " 'alexgand/springer_free_books',\n",
    " 'etalab/covid19-dashboard',\n",
    " 'soroushchehresa/awesome-coronavirus',\n",
    " 'ieee8023/covid-chestxray-dataset',\n",
    " 'covid19india/covid19india',\n",
    " 'Priesemann-Group/covid19_inference',\n",
    " '3778/COVID-19',\n",
    " 'nihp-public/covid19-app-system-public',\n",
    " 'PubInv/covid19-vent-list',\n",
    " 'Kamaropoulos/COVID19Py',\n",
    " 'leafcoder/django-covid19',\n",
    " 'lestweforget/COVID-19-Timeline',\n",
    " 'VictimOfMaths/COVID-19',\n",
    " 'nthopinion/covid19',\n",
    " 'contactmodel/COVID19-Japan-Reff',\n",
    " 'sambreen27/covid19',\n",
    " 'datacamp/COVID-19-EDA-tutorial',\n",
    " 'yunwei37/COVID-19-NLP-vis',\n",
    " 'Rank23/COVID19',\n",
    " 'virtual-biohackathons/covid-19-bh20',\n",
    " 'localeai/covid19-live-visualization']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "d64e31e7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Unicorndy/covid19_dashboard',\n",
       " 'twiecki/covid19',\n",
       " 'phildini/stayinghomeclub',\n",
       " 'globalcitizen/2019-wuhan-coronavirus-data',\n",
       " 'greenelab/covid19-review',\n",
       " 'covid19-forecast-hub-europe/covid19-forecast-hub-europe',\n",
       " 'minhealthnz/nz-covid-data',\n",
       " 'closedloop-ai/cv19index',\n",
       " 'jannisborn/covid19_ultrasound',\n",
       " 'tomwhite/covid-19-uk-data',\n",
       " 'elcronos/COVID-19',\n",
       " 'muhammedtalo/COVID-19',\n",
       " 'trekhleb/covid-19',\n",
       " 'tirthajyoti/Covid-19-analysis',\n",
       " 'devarthurribeiro/covid19-brazil-api',\n",
       " 'dswh/voila-covid-19-dashboard',\n",
       " 'nihp-public/COVID-19-app-iOS-BETA',\n",
       " 'chschoenenberger/covid19_dashboard',\n",
       " 'midudev/covid-19-spread-simulator',\n",
       " 'BDI-pathogens/OpenABM-Covid19',\n",
       " 'wareneutron/covid19-tracker-cli',\n",
       " 'hostolab/covidliste',\n",
       " 'imtsuki/bupt-ncov-report-action',\n",
       " 'heremaps/here-covid-19-tracker',\n",
       " 'Institut-Zdravotnych-Analyz/covid19-data',\n",
       " 'microsoft/Bing-COVID-19-Data',\n",
       " 'Priesemann-Group/covid19_inference_forecast',\n",
       " 'alexgand/springer_free_books',\n",
       " 'etalab/covid19-dashboard',\n",
       " 'soroushchehresa/awesome-coronavirus',\n",
       " 'ieee8023/covid-chestxray-dataset',\n",
       " 'covid19india/covid19india',\n",
       " 'Priesemann-Group/covid19_inference',\n",
       " '3778/COVID-19',\n",
       " 'nihp-public/covid19-app-system-public',\n",
       " 'PubInv/covid19-vent-list',\n",
       " 'Kamaropoulos/COVID19Py',\n",
       " 'leafcoder/django-covid19',\n",
       " 'lestweforget/COVID-19-Timeline',\n",
       " 'VictimOfMaths/COVID-19',\n",
       " 'nthopinion/covid19',\n",
       " 'contactmodel/COVID19-Japan-Reff',\n",
       " 'sambreen27/covid19',\n",
       " 'datacamp/COVID-19-EDA-tutorial',\n",
       " 'yunwei37/COVID-19-NLP-vis',\n",
       " 'Rank23/COVID19',\n",
       " 'virtual-biohackathons/covid-19-bh20',\n",
       " 'localeai/covid19-live-visualization']"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "1b9cb7db",
   "metadata": {},
   "outputs": [],
   "source": [
    "REPOS.extend(bill)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "181efb68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "138"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(REPOS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "d456fb3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "more=['eAzure/COVID-19-Data',\n",
    " 'rlindskog/covid19-graphql',\n",
    " 'rozierguillaume/covid-19',\n",
    " 'simonblowsnow/COVID-19-Visualization',\n",
    " 'oscarhealth/covid19-testing',\n",
    " 'lopezbec/COVID19_Tweets_Dataset',\n",
    " 'covid19india/CovidCrowd',\n",
    " 'covid-19-net/covid-19-community',\n",
    " 'lisphilar/covid19-sir',\n",
    " 'CDCgov/covid19healthbot',\n",
    " 'covidcaremap/covid19-healthsystemcapacity',\n",
    " 'minvws/nl-covid19-notification-app-website',\n",
    " 'celiavelmar/open-covid19-test',\n",
    " 'wal33d006/novel_covid_19',\n",
    " 'KeithGalli/generate-analytics-report',\n",
    " 'google-research/open-covid-19-data',\n",
    " 'PatilShreyas/Covid19-Notifier-IN',\n",
    " 'peixebabel/COVID-19',\n",
    " 'Altevir/xamarinforms-covid19',\n",
    " 'cojocchen/covid19_critically_ill',\n",
    " 'OssamaRafique/Covid-19-Statistics-Dashboard-Angular-9',\n",
    " 'CodeExplainedRepo/COVID-19_TRACKER_HTML_CSS_JAVASCRIPT',\n",
    " 'covid19-data/covid19-data',\n",
    " 'MickeysClubhouse/COVID-19-rumor-dataset',\n",
    " 'Glacier-Ice/Covid-19-data-science',\n",
    " 'bhattbhavesh91/cowin-vaccination-slot-availability',\n",
    " 'skyerhxx/COVID-19_Tracking',\n",
    " 'jihoo-kim/Data-Science-for-COVID-19',\n",
    " 'aboullaite/Covid19-MA',\n",
    " 'minvws/nl-covid19-notification-app-design',\n",
    " 'joanby/covid19',\n",
    " 'covid19cz/erouska-android',\n",
    " 'whhsky/COVID-19-KSH',\n",
    " 'Laeyoung/COVID-19-API',\n",
    " 'code4sabae/covid19',\n",
    " 'pennsignals/chime',\n",
    " 'CITF-Malaysia/citf-public',\n",
    " 'covid19cubadata/covid19cubadata.github.io',\n",
    " 'frankkramer-lab/covid19.MIScnn',\n",
    " 'jgehrcke/covid-19-germany-gae',\n",
    " 'kawalcovid19/wargabantuwarga.com',\n",
    " 'helpwithcovid/covid-volunteers',\n",
    " 'HarshdeepKahlon/hiring2020',\n",
    " 'nesk/covid-19-certificate',\n",
    " 'M-Media-Group/Covid-19-API',\n",
    " 'hifiaz/covid19',\n",
    " 'covid-19-au/covid-19-au.github.io',\n",
    " 'microsoft/COVID-19-Widget',\n",
    " 'MohGovIL/hamagen-react-native',\n",
    " 'neherlab/covid19_scenarios_data',\n",
    " 'umanghome/covid19-twitter.in',\n",
    " 'covidatlas/coronadatascraper',\n",
    " 'mrc-ide/COVID19_CFR_submission',\n",
    " 'imdevskp/covid-19-india-data',\n",
    " 'ryo-ma/covid19-japan-web-api',\n",
    " 'jorgeperezrojas/covid19-data',\n",
    " 'publichealthengland/coronavirus-dashboard',\n",
    " 'sfu-db/covid19-datasets',\n",
    " 'arpanmangal/CovidAID',\n",
    " 'descarteslabs/DL-COVID-19',\n",
    " 'DataHerb/dataset-covid-19',\n",
    " 'Coders-Of-XDA-OT/covid19-status-android',\n",
    " 'USCreditCardGuide/airlines-to-china-covid-19',\n",
    " 'ThisIsIsaac/Data-Science-for-COVID-19',\n",
    " 'Reynadi531/api-covid19-indonesia-v2',\n",
    " 'InstituteforDiseaseModeling/covasim',\n",
    " 'Anthonyeef/covid-19-bitbar-plugin',\n",
    " 'geohot/corona',\n",
    " 'idandrd/israel-covid19-data',\n",
    " 'HariKulhari06/Covid-19-Tracker',\n",
    " 'maykbrito/covid-19',\n",
    " 'cagov/covid19',\n",
    " 'rpandey1234/Covid19Tracker',\n",
    " 'midudev/covid-vacuna',\n",
    " 'sassoftware/covid-19-sas',\n",
    " 'JieYingWu/COVID-19_US_County-level_Summaries',\n",
    " 'kiang/covid19',\n",
    " 'HAIRLAB/Pre_Surv_COVID_19',\n",
    " 'vinitshahdeo/covid19api',\n",
    " 'rphl/corona-widget',\n",
    " 'bradtraversy/vue-covid-tracker']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "8246729f",
   "metadata": {},
   "outputs": [],
   "source": [
    "REPOS.extend(more)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "ee555d12",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "219"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(REPOS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "48d59847",
   "metadata": {},
   "outputs": [],
   "source": [
    "REPOS=set(REPOS)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d999f6c2",
   "metadata": {},
   "source": [
    "# File Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "fce95a5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "from typing import Dict, List, Optional, Union, cast\n",
    "import requests\n",
    "\n",
    "from env import github_token, github_username\n",
    "\n",
    "# TODO: Make a github personal access token.\n",
    "#     1. Go here and generate a personal access token: https://github.com/settings/tokens\n",
    "#        You do _not_ need select any scopes, i.e. leave all the checkboxes unchecked\n",
    "#     2. Save it in your env.py file under the variable `github_token`\n",
    "# TODO: Add your github username to your env.py file under the variable `github_username`\n",
    "# TODO: Add more repositories to the `REPOS` list below.\n",
    "\n",
    "#REPOS = [\n",
    "#    REPOS[:5],\n",
    "#]\n",
    "\n",
    "headers = {\"Authorization\": f\"token {github_token}\", \"User-Agent\": github_username}\n",
    "\n",
    "if headers[\"Authorization\"] == \"token \" or headers[\"User-Agent\"] == \"\":\n",
    "    raise Exception(\n",
    "        \"You need to follow the instructions marked TODO in this script before trying to use it\"\n",
    "    )\n",
    "\n",
    "\n",
    "def github_api_request(url: str) -> Union[List, Dict]:\n",
    "    response = requests.get(url, headers=headers)\n",
    "    response_data = response.json()\n",
    "    if response.status_code != 200:\n",
    "        raise Exception(\n",
    "            f\"Error response from github api! status code: {response.status_code}, \"\n",
    "            f\"response: {json.dumps(response_data)}\"\n",
    "        )\n",
    "    return response_data\n",
    "\n",
    "\n",
    "def get_repo_language(repo: str) -> str:\n",
    "    url = f\"https://api.github.com/repos/{repo}\"\n",
    "    repo_info = github_api_request(url)\n",
    "    if type(repo_info) is dict:\n",
    "        repo_info = cast(Dict, repo_info)\n",
    "        if \"language\" not in repo_info:\n",
    "            raise Exception(\n",
    "                \"'language' key not round in response\\n{}\".format(json.dumps(repo_info))\n",
    "            )\n",
    "        return repo_info[\"language\"]\n",
    "    raise Exception(\n",
    "        f\"Expecting a dictionary response from {url}, instead got {json.dumps(repo_info)}\"\n",
    "    )\n",
    "\n",
    "\n",
    "def get_repo_contents(repo: str) -> List[Dict[str, str]]:\n",
    "    url = f\"https://api.github.com/repos/{repo}/contents/\"\n",
    "    contents = github_api_request(url)\n",
    "    if type(contents) is list:\n",
    "        contents = cast(List, contents)\n",
    "        return contents\n",
    "    raise Exception(\n",
    "        f\"Expecting a list response from {url}, instead got {json.dumps(contents)}\"\n",
    "    )\n",
    "\n",
    "\n",
    "def get_readme_download_url(files: List[Dict[str, str]]) -> str:\n",
    "    \"\"\"\n",
    "    Takes in a response from the github api that lists the files in a repo and\n",
    "    returns the url that can be used to download the repo's README file.\n",
    "    \"\"\"\n",
    "    for file in files:\n",
    "        if file[\"name\"].lower().startswith(\"readme\"):\n",
    "            return file[\"download_url\"]\n",
    "    raise Exception(\"README file not found in repository\")\n",
    "\n",
    "\n",
    "def process_repo(repo: str) -> Dict[str, str]:\n",
    "    \"\"\"\n",
    "    Takes a repo name like \"gocodeup/codeup-setup-script\" and returns a\n",
    "    dictionary with the language of the repo and the readme contents.\n",
    "    \"\"\"\n",
    "    contents = get_repo_contents(repo)\n",
    "    try:\n",
    "        readme_download_url = get_readme_download_url(contents)\n",
    "        readme_contents = requests.get(readme_download_url).text\n",
    "    except Exception:\n",
    "        readme_contents = \"\"\n",
    "    return {\n",
    "        \"repo\": repo,\n",
    "        \"language\": get_repo_language(repo),\n",
    "        \"readme_contents\": readme_contents,\n",
    "    }\n",
    "\n",
    "\n",
    "def scrape_github_data() -> List[Dict[str, str]]:\n",
    "    \"\"\"\n",
    "    Loop through all of the repos and process them. Returns the processed data.\n",
    "    \"\"\"\n",
    "    return [process_repo(repo) for repo in REPOS]\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    data = scrape_github_data()\n",
    "    json.dump(data, open(\"data.json\", \"w\"), indent=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbc02ac2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}