1.1_ape_dirtyexplospacy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matlex propose la vente et l’installation de poèles, d’inserts et de cheminées à bois ou granules à mérignies dans 100% du nord\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\n",
    "from sklearn.base import TransformerMixin\n",
    "from sklearn.pipeline import Pipeline\n",
    "import spacy\n",
    "\n",
    "\n",
    "# Load the small English model\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "\n",
    "# Process a text\n",
    "doc = nlp(\"She ate the pizza\")\n",
    "\n",
    "text = \"It’s official: Apple is the first U.S. public company to reach a $1 trillion market value\"\n",
    "\n",
    "# Process the text\n",
    "doc = nlp(text)\n",
    "\n",
    "# Print the document text\n",
    "print(doc.text)\n",
    "\n",
    "# Iterate over the tokens\n",
    "#for token in doc:\n",
    "    # Print the text and the predicted part-of-speech tag\n",
    "#    print(token.text, token.pos_)\n",
    "\n",
    "# Iterate over the predicted entities\n",
    "for ent in doc.ents :\n",
    "    # Print the entity text and its label\n",
    "    print(ent.label_, ent.text)\n",
    "# Initialize the matcher with the shared vocab\n",
    "matcher = Matcher(nlp.vocab)\n",
    "\n",
    "# Add the pattern to the matcher\n",
    "pattern = [{'TEXT': 'iPhone'}, {'TEXT': 'X'}]\n",
    "matcher.add('IPHONE_PATTERN', None, pattern)\n",
    "\n",
    "# Process some text\n",
    "doc = nlp(\"New iPhone X release date leaked\")\n",
    "\n",
    "# Call the matcher on the doc\n",
    "matches = matcher(doc)\n",
    "#from spacy.lang.fr import French\n",
    "#nlp = French()\n",
    "doc = nlp(\"matlex propose la vente et l’installation de poèles, d’inserts et de cheminées à bois ou granules à mérignies dans 100% du nord\")\n",
    "\n",
    "# Iterate over the tokens in the doc\n",
    "for token in doc:\n",
    "    # Check if the token resembles a number\n",
    "    if token.like_num:\n",
    "        # Get the next token in the document\n",
    "        next_token = doc[token.i+1]\n",
    "        # Check if the next token's text equals '%'\n",
    "        if next_token.text == \"%\":\n",
    "            print(\"Percentage found:\", token.text)\n",
    "\"\"\"\n",
    "\n",
    "print(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "It’s official: Apple is the first U.S. public company to reach a $1 trillion market value\n",
      "ORG Apple\n",
      "ORDINAL first\n",
      "GPE U.S.\n",
      "MONEY $1 trillion\n"
     ]
    }
   ],
   "source": [
    "import string\n",
    "from spacy.lang.fr.stop_words import STOP_WORDS\n",
    "from spacy.lang.fr import French\n",
    "\n",
    "# \n",
    "# First step : construct the dataframe\n",
    "\n",
    "# 2nd step : clean dataframe\n",
    "\n",
    "# 3ird : vectorisation\n",
    "bow_vector = CountVectorizer(tokenizer = spacy_tokenizer, ngram_range=(1,1))\n",
    "\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\n",
    "from sklearn.base import TransformerMixin\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "# Create our list of punctuation marks\n",
    "punctuations = string.punctuation\n",
    "\n",
    "# Create our list of stopwords\n",
    "nlp = spacy.load('fr')\n",
    "stop_words = spacy.lang.fr.stop_words.STOP_WORDS\n",
    "\n",
    "# Load English tokenizer, tagger, parser, NER and word vectors\n",
    "parser = French()\n",
    "\n",
    "# Creating our tokenizer function\n",
    "def spacy_tokenizer(sentence):\n",
    "    # Creating our token object, which is used to create documents with linguistic annotations.\n",
    "    mytokens = parser(sentence)\n",
    "\n",
    "    # Lemmatizing each token and converting each token into lowercase\n",
    "    mytokens = [ word.lemma_.lower().strip() if word.lemma_ != \"-PRON-\" else word.lower_ for word in mytokens ]\n",
    "\n",
    "    # Removing stop words\n",
    "    mytokens = [ word for word in mytokens if word not in stop_words and word not in punctuations ]\n",
    "\n",
    "    # return preprocessed list of tokens\n",
    "    return mytokens\n",
    "\n",
    "# Custom transformer using spaCy\n",
    "class predictors(TransformerMixin):\n",
    "    def transform(self, X, **transform_params):\n",
    "        # Cleaning Text\n",
    "        return [clean_text(text) for text in X]\n",
    "\n",
    "    def fit(self, X, y=None, **fit_params):\n",
    "        return self\n",
    "\n",
    "    def get_params(self, deep=True):\n",
    "        return {}\n",
    "\n",
    "# Basic function to clean the text\n",
    "def clean_text(text):\n",
    "    # Removing spaces and converting text into lowercase\n",
    "    return text.strip().lower()\n",
    "\n",
    "bow_vector = CountVectorizer(tokenizer = spacy_tokenizer, ngram_range=(1,1))\n",
    "tfidf_vector = TfidfVectorizer(tokenizer = spacy_tokenizer)\n",
    "\n",
    "\n",
    "Splitting The Data into Training and Test Sets\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = df_amazon['verified_reviews'] # the features we want to analyze\n",
    "ylabels = df_amazon['feedback'] # the labels, or answers, we want to test against\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, ylabels, test_size=0.3)\n",
    "\n",
    "maphrase = \"matlex propose la vente et l’installation de poèles, d’inserts et de cheminées à bois ou granules à mérignies dans 100% du nord\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}