Skip to content

Commit d83bedc

Browse files
omerertomerert
committed
Created a new Python file that will include functions for bias detection specifically for text. Also added content for success page for text (only tuples for rating)
1 parent 31c9a7d commit d83bedc

File tree

7 files changed

+201
-22
lines changed

7 files changed

+201
-22
lines changed

‎db.sqlite3

0 Bytes
Binary file not shown.

‎nlpFiles/detectionInText.py

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
from transformers import AutoTokenizer, TFAutoModelForSequenceClassification, pipeline
2+
from nltk.tokenize import sent_tokenize
3+
from selenium import webdriver
4+
import os
5+
from bs4 import BeautifulSoup
6+
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
7+
8+
def biasInText(userText) -> list:
9+
# Get models
10+
tokenizer = AutoTokenizer.from_pretrained("d4data/bias-detection-model")
11+
model = TFAutoModelForSequenceClassification.from_pretrained("d4data/bias-detection-model")
12+
classifier = pipeline('text-classification', model=model, tokenizer=tokenizer)
13+
analyzer = SentimentIntensityAnalyzer()
14+
resultList = []
15+
# Separate user text by sentences
16+
sentences = sent_tokenize(userText)
17+
for sent in sentences:
18+
resultList.append(f"{classifier(sent)[0]}")
19+
return resultList
20+
# print(biasInText("My name is Omer. I am testing the new bias detection in text. Hopefully it works."))

‎nlpFiles/getTextFromWeb.py

+20-11
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,8 @@
1-
import nltk
2-
import transformers
31
from transformers import AutoTokenizer, TFAutoModelForSequenceClassification, pipeline
4-
from nltk.tokenize import sent_tokenize, word_tokenize
5-
import urllib.request
2+
from nltk.tokenize import sent_tokenize
63
from selenium import webdriver
74
import os
85
from bs4 import BeautifulSoup
9-
from nltk.corpus import wordnet as wn
106
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
117

128
# class BiasDetection:
@@ -51,9 +47,7 @@
5147
# soupText = BeautifulSoup(pageSource, 'html.parser')
5248
# wantedText = soupText.findAll("p")
5349
# return wantedText
54-
#!!!!! IMPORTANT !!!!!
55-
# GOING TO REWORK BIAS DETECTION TO ACTUALLY DETECT BIAS, CONCERNING WHERE IT IS IN THE TEXT AND IT'S TYPE(BIAS TOWARDS WHO?, WHAT KIND OF BIAS?, ETC)
56-
#!!!!! IMPORTANT !!!!!
50+
5751

5852
def percentToInt(list):
5953
# Remove '%' from strings and convert to float
@@ -127,12 +121,11 @@ def highRatedSent(paraList):
127121
return highList
128122

129123
def polarityRating(list, pageSource):
130-
# Starting up our models and analyzers
124+
# Starting up our models
131125
tokenizer = AutoTokenizer.from_pretrained("d4data/bias-detection-model")
132126
model = TFAutoModelForSequenceClassification.from_pretrained("d4data/bias-detection-model")
133127
classifier = pipeline('text-classification', model=model, tokenizer=tokenizer)
134128
analyzer = SentimentIntensityAnalyzer()
135-
136129
wantedText = getParagraph(pageSource)
137130
# Declare variables for average polarity
138131
count = 0
@@ -170,4 +163,20 @@ def polarityRating(list, pageSource):
170163
overallList = [str(round(totBias / count * 100, 1)) + "%", str(round(totNon / count * 100, 1)) + "%", str(round(totPos / polarityCount * 100, 1)) + "%", str(round(totNeg / polarityCount * 100, 1)) + "%" , round(totNeg / polarityCount * 100, 1)]
171164
overallList.append(round(totBias / count * 100, 0))
172165
if count != 0:
173-
list.insert(0,overallList)
166+
list.insert(0,overallList)
167+
# def getText(pageSource):
168+
# tokenizer = AutoTokenizer.from_pretrained("d4data/bias-detection-model")
169+
# model = TFAutoModelForSequenceClassification.from_pretrained("d4data/bias-detection-model")
170+
# classifier = pipeline('text-classification', model=model, tokenizer=tokenizer)
171+
# # analyzer = SentimentIntensityAnalyzer()
172+
173+
# wantedText = getParagraph(pageSource)
174+
# for para in wantedText:
175+
# sentences = sent_tokenize(para.text.strip())
176+
# for sent in sentences:
177+
# if sent != "":
178+
# print(classifier(sent)[0])
179+
# getText(webScrape("https://www.breitbart.com/2024-election/2024/04/11/doj-unmasks-inconsistencies-in-fani-williss-federal-grant-funds-use/"))
180+
# list1 = []
181+
# polarityRating(list1, webScrape("https://www.breitbart.com/2024-election/2024/04/11/doj-unmasks-inconsistencies-in-fani-williss-federal-grant-funds-use/"))
182+
# print(list1)

‎nlpFiles/trainedModel.py

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
def model():
2+
# Starting up our models and analyzers
3+
tokenizer = AutoTokenizer.from_pretrained("d4data/bias-detection-model")
4+
model = TFAutoModelForSequenceClassification.from_pretrained("d4data/bias-detection-model")
5+
return pipeline('text-classification', model=model, tokenizer=tokenizer)
6+

‎nobiasapp/templates/successPage.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@
129129

130130
</label>
131131
<nav class="navbar">
132-
<a href="../" style="--i:0;">Home<span></span></a>
132+
<a href="home" style="--i:0;">Home<span></span></a>
133133
<a href="aboutus" style="--i:1;">About Us<span></span></a>
134134
<a href="#" style="--i:2;"> Contacts<span></span></a>
135135
<a href="tech" style="--i:3;"> How We Detect Bias<span></span></a>

‎nobiasapp/templates/textPage.html

+150-8
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,154 @@
11
{% load static %}
2+
23
<!DOCTYPE html>
34
<html lang="en">
4-
<head>
5-
<meta charset="UTF-8">
6-
<meta name="viewport" content="width=device-width, initial-scale=1.0">
7-
<title>Document</title>
8-
</head>
9-
<body>
10-
<h1> {{ thisVar }}</h1>
11-
</body>
5+
6+
<head>
7+
<meta charset="UTF-8">
8+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
9+
<title>Successfuly Submitted Text</title>
10+
<link rel="stylesheet" href="{% static 'css/style.css' %}">
11+
</head>
12+
<style>
13+
.logo img {
14+
width: 100px;
15+
float: left;
16+
padding-left: 10%;
17+
padding-top: 2%;
18+
}
19+
canvas {
20+
position: relative;
21+
top: 50%;
22+
left: 50%;
23+
transform: translate(-50%, -50%);
24+
}
25+
.text {
26+
color: white;
27+
font-weight: 600;
28+
font-size: 24px;
29+
position: absolute;
30+
bottom: 55px;
31+
left: 50%;
32+
transform: translate(-50%, -50%);
33+
}
34+
.helpstuff {
35+
display: flex;
36+
margin:auto;
37+
width:500px;
38+
}
39+
40+
.tooltip {
41+
position: relative;
42+
display: inline-block;
43+
}
44+
45+
.tooltip .tooltiptext {
46+
visibility: hidden;
47+
width: 200px;
48+
background-color: white;
49+
background: #415f77;
50+
color: #fff;
51+
text-align: center;
52+
padding: 5px 0;
53+
border-radius: 6px;
54+
opacity:0.77;
55+
56+
position: absolute;
57+
position: right;
58+
z-index: 1;
59+
}
60+
61+
.tooltip:hover .tooltiptext {
62+
visibility: visible;
63+
}
64+
65+
.hoverstuff {
66+
display: flex;
67+
align-items: center;
68+
justify-content: center;
69+
}
70+
71+
.box {
72+
background-color: #f8f8f8;
73+
padding: 20px;
74+
border-radius: 10px;
75+
box-shadow: 0 0 20px rgba(0, 0, 0, 0.1);
76+
line-height: 30px;
77+
padding-top: 25px;
78+
}
79+
80+
.polarity-bar {
81+
width: 100%;
82+
height: 20px;
83+
background-color: #ddd;
84+
border-radius: 5px;
85+
overflow: hidden;
86+
--cwidth1: {{ thisVar.0.0 }};
87+
--cwidth2: {{ thisVar.0.1 }};
88+
--cwidth3: {{ thisVar.0.2 }};
89+
}
90+
91+
.polarity-bar .section {
92+
height: 100%;
93+
float: left;
94+
}
95+
96+
.red-section {
97+
background-color: #ff4646;
98+
width: var(--cwidth1);
99+
}
100+
101+
.grey-section {
102+
background-color: #a0a0a0;
103+
width: var(--cwidth2);
104+
}
105+
106+
.green-section {
107+
background-color: #4caf50;
108+
width: var(--cwidth3);
109+
}
110+
h2{
111+
color: black;
112+
}
113+
114+
</style>
115+
116+
<body>
117+
<header class="header">
118+
119+
<div class="logo">
120+
<img src="{% static 'img/logo.png' %}" alt="1st-logo-1" style="scale: 0.8;">
121+
</div>
122+
123+
124+
<input type="checkbox" id="check">
125+
<label for="check" class="icons">
126+
<i class="bx bx-menu" id="menu-icon"></i>
127+
<i class="bx bx-x" id="close-icon"></i>
128+
129+
130+
</label>
131+
<nav class="navbar">
132+
<a href="home" style="--i:0;">Home<span></span></a>
133+
<a href="aboutus" style="--i:1;">About Us<span></span></a>
134+
<a href="#" style="--i:2;"> Contacts<span></span></a>
135+
<a href="tech" style="--i:3;"> How We Detect Bias<span></span></a>
136+
</nav>
137+
</header>
138+
<br><br>
139+
<div class="box">
140+
141+
<ul style="list-style-type: none; padding: 0; ">
142+
143+
{% for link in sentences %}
144+
<li style="font-family: 'Open Sans', sans-serif; color: #003366; word-wrap: break-word;line-height: 40px;">
145+
{{ link }}
146+
</li>
147+
{% endfor %}
148+
</ul>
149+
</div>
150+
151+
152+
</body>
153+
12154
</html>

‎nobiasapp/views.py

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from django.shortcuts import render
22
from nlpFiles.getTextFromWeb import polarityRating, highRatedSent, getTitle, webScrape
3+
from nlpFiles.detectionInText import biasInText
34
from .forms import SubmitLinkForm, SubmitTextBoxForm
45

56
# def submitLink(request):
@@ -37,12 +38,13 @@ def home(request):
3738
title = getTitle(pageSource)
3839
polarityRating(paragraph, pageSource)
3940
highValuedList = highRatedSent(paragraph)
40-
return render(request, 'successPage.html', {'thisVar': paragraph, "highValuedList": highValuedList, 'title': title})
41+
return render(request, 'successPage.html', {'thisVar': paragraph, "highValuedList": highValuedList, 'title': title}) #Redirects to successpage when a link is submitted
4142

4243
elif text_form.is_valid():
4344
text_object = text_form.save()
45+
sentences = biasInText(str(text_object))
4446
# Do something with the text form submission
45-
return render(request, 'textPage.html', {'thisVar': text_object}) # Render a success page for text form submission
47+
return render(request, 'textPage.html', {'sentences': sentences}) #Redirects to the successpage when text is submitted
4648

4749
else:
4850
link_form = SubmitLinkForm()

0 commit comments

Comments
 (0)