Update user study results; add statistical tests

user_study/results_2023-02-01-9pm.txt

@@ -1,5 +1,5 @@
-Percentage of users with negative opinion of principal
+Number of users with negative, positive opinion of principal
 ------------------
 
-test    = 37.9% (11/29)
-control = 25.9% (7/27)
+test    = 11, 18
+control = 07, 20

user_study/results_2023-02-02-3pm.txt

@@ -0,0 +1,5 @@
+Number of users with negative, positive opinion of principal
+------------------
+
+test    = 11, 18
+control = 07, 22

user_study/results_2023-02-02-3pm_sanitized.txt

@@ -0,0 +1,14 @@
+Number of users with negative, positive opinion of principal
+------------------
+
+test    = 11, 16
+control = 04, 21
+
+
+
+removed Control #23
+removed Test #15
+removed Test #06
+removed Control #02
+removed Control #22
+removed Control #27

user_study/results_2023-02-02-6pm_sanitized.txt

@@ -0,0 +1,14 @@
+Number of users with negative, positive opinion of principal
+------------------
+
+test    = 11, 17
+control = 04, 21
+
+
+
+removed Control #23
+removed Test #15
+removed Test #06
+removed Control #02
+removed Control #22
+removed Control #27

user_study/statistical_tests.py

@@ -0,0 +1,22 @@
+from scipy import stats
+
+
+# run statistical test on user study results
+
+with open("results_2023-02-02-6pm_sanitized.txt") as f:
+	results = f.readlines()[3:]
+	test = results[0]
+	control = results[1]
+
+	test_neg = int(test[10:12])
+	test_pos = int(test[14:16])
+	control_neg = int(control[10:12])
+	control_pos = int(control[14:16])
+
+	contingency_table = [[test_neg, control_neg], [test_pos, control_pos]]
+
+	res = stats.boschloo_exact(contingency_table, alternative="greater", n=256)
+	# res = stats.barnard_exact(contingency_table, alternative="greater", pooled=True, n=256)  # an alternative test that is also statistically valid but generally not as powerful
+
+	pvalue = round(res.pvalue, 4)
+	print(f"p={pvalue}")