-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathbiased_logistic.py
135 lines (111 loc) · 3.92 KB
/
biased_logistic.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
import numpy as np
from scipy import optimize
from scipy.special import logsumexp
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelBinarizer
from sklearn.utils.extmath import safe_sparse_dot, squared_norm
from sklearn.utils.optimize import _check_optimize_result
# from sklearn.linear_model._logistic import (
# #LabelBinarizer,
# #_check_optimize_result,
# #logsumexp,
# #safe_sparse_dot,
# #squared_norm,
# )
def loss_and_grad(x, *args):
return _multinomial_loss_grad(x, *args)[0:2]
def _multinomial_loss_grad(w, X, Y, alpha, bias_w=0, sample_weight=None):
n_classes = Y.shape[1]
n_features = X.shape[1]
if sample_weight is None:
sample_weight = np.ones(X.shape[0])
fit_intercept = w.size == n_classes * (n_features + 1)
grad = np.zeros((n_classes, n_features + bool(fit_intercept)), dtype=X.dtype)
loss, p, w = _multinomial_loss(w, X, Y, alpha, sample_weight, bias_w)
sample_weight = sample_weight[:, np.newaxis]
diff = sample_weight * (p - Y)
grad[:, :n_features] = safe_sparse_dot(diff.T, X)
grad[:, :n_features] += alpha * (w - bias_w)
if fit_intercept:
grad[:, -1] = diff.sum(axis=0)
return loss, grad.ravel(), p
def _multinomial_loss(w, X, Y, alpha, sample_weight, bias_w):
n_classes = Y.shape[1]
n_features = X.shape[1]
fit_intercept = w.size == (n_classes * (n_features + 1))
w = w.reshape(n_classes, -1)
sample_weight = sample_weight[:, np.newaxis]
if fit_intercept:
intercept = w[:, -1]
w = w[:, :-1]
else:
intercept = 0
p = safe_sparse_dot(X, w.T)
p += intercept
p -= logsumexp(p, axis=1)[:, np.newaxis]
loss = -(sample_weight * Y * p).sum()
loss += 0.5 * alpha * squared_norm(w - bias_w)
p = np.exp(p, p)
return loss, p, w
def solve_LR(
X,
y,
Cs,
sample_weight=None,
tol=1e-4,
max_iter=100,
verbose=0,
w0=None,
bias_w=0,
intercept=False,
):
coefs = list()
n_iter = np.zeros(len(Cs), dtype=np.int32)
classes = np.unique(y)
n_sample, n_feat = X.shape
if intercept:
X = np.concatenate([X, np.ones((n_sample, 1))], axis=1)
if isinstance(bias_w, np.ndarray):
bias_w = np.concatenate([bias_w, np.zeros((bias_w.shape[0], 1))], axis=1)
if w0 is None:
w0 = np.zeros([classes.size, X.shape[1]])
elif intercept:
w0 = np.concatenate([w0, np.zeros((w0.shape[0], 1))], axis=1)
lbin = LabelBinarizer()
Y_multi = lbin.fit_transform(y)
if Y_multi.shape[1] == 1:
Y_multi = np.hstack([1 - Y_multi, Y_multi])
for i, C in enumerate(Cs):
iprint = [-1, 50, 1, 100, 101][np.searchsorted(np.array([0, 1, 2, 3]), verbose)]
opt_res = optimize.minimize(
loss_and_grad,
w0,
method="L-BFGS-B",
jac=True,
args=(X, Y_multi, C, bias_w, sample_weight),
options={"iprint": iprint, "gtol": tol, "maxiter": max_iter},
)
n_iter_i = _check_optimize_result("lbfgs", opt_res, max_iter, extra_warning_msg=None)
w0, loss = opt_res.x, opt_res.fun
n_classes = max(2, classes.size)
multi_w0 = np.reshape(w0, (n_classes, -1))
if n_classes == 2:
multi_w0 = multi_w0[1][np.newaxis, :]
coefs.append(multi_w0.copy())
n_iter[i] = n_iter_i
return np.array(coefs), np.array(Cs), n_iter
def create_LR_object(coef, n_class, n_iter, max_iter=100, tol=1e-4, intercept=False):
lr = LogisticRegression(max_iter=max_iter, multi_class="multinomial", fit_intercept=intercept, tol=tol)
if intercept:
w0 = coef[:, :-1]
b = coef[:, -1]
else:
w0 = coef
b = 0
class_labels = np.arange(n_class)
lr.coef_ = w0
lr.intercept_ = b
lr.n_iter_ = n_iter
lr.classes_ = class_labels
# lr = lr.set_params(coef_=w0, intercept_=b, n_iter_=n_iter, classes_=class_labels)
return lr