1"""
2This module provides different metrics for evaluation.
3"""
4import numpy as np
5from sklearn.metrics import roc_auc_score as skelarn_roc_auc_score, f1_score as skelarn_f1_scpre, \
6 mean_absolute_error, root_mean_squared_error, r2_score as sklearn_r2_score
7
8
[docs]
9def r2_score(y_pred: np.ndarray, y: np.ndarray) -> float:
10 """
11 Computes the R^2 score (also called the coefficient of determination).
12
13 Parameters
14 ----------
15 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
16 Predicted outputs.
17 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
18 Ground truth outputs.
19
20 Returns
21 -------
22 `float`
23 R^2 score.
24 """
25 return sklearn_r2_score(y, y_pred)
26
27
[docs]
28def running_r2_score(y_pred: np.ndarray, y: np.ndarray) -> list[float]:
29 """
30 Computes and returns the running R^2 score -- i.e. the R^2 score for every point in time.
31
32 Parameters
33 ----------
34 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
35 Predicted outputs.
36 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
37 Ground truth outputs.
38
39 Returns
40 -------
41 `list[float]`
42 The running R^2 score.
43 """
44 r = []
45
46 for t in range(2, len(y_pred)):
47 r.append(r2_score(y_pred[:t], y[:t]))
48
49 return r
50
51
[docs]
52def mean_squared_error(y_pred: np.ndarray, y: np.ndarray) -> float:
53 """
54 Computes the Mean Squared Error (MSE).
55
56 Parameters
57 ----------
58 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
59 Predicted outputs.
60 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
61 Ground truth outputs.
62
63 Returns
64 -------
65 `float`
66 MSE.
67 """
68 return root_mean_squared_error(y, y_pred)**2
69
70
[docs]
71def running_mse(y_pred: np.ndarray, y: np.ndarray) -> list[float]:
72 """
73 Computes the running Mean Squared Error (MSE) -- i.e. the MSE for every point in time.
74
75 Parameters
76 ----------
77 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
78 Predicted outputs.
79 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
80 Ground truth outputs.
81
82 Returns
83 -------
84 `float`
85 Running MSE.
86 """
87 if not isinstance(y_pred, np.ndarray):
88 raise TypeError("'y_pred' must be an instance of 'numpy.ndarray' " +
89 f"but not of '{type(y_pred)}'")
90 if not isinstance(y, np.ndarray):
91 raise TypeError("'y' must be an instance of 'numpy.ndarray' " +
92 f"but not of '{type(y)}'")
93 if y_pred.shape != y.shape:
94 raise ValueError(f"Shape mismatch: {y_pred.shape} vs. {y.shape}")
95 if len(y_pred.shape) != 1:
96 raise ValueError("'y_pred' must be a 1d array")
97 if len(y.shape) != 1:
98 raise ValueError("'y' must be a 1d array")
99
100 e_sq = np.square(y - y_pred)
101 r_mse = list(esq for esq in e_sq)
102
103 for i in range(1, len(y)):
104 r_mse[i] = float((i * r_mse[i - 1]) / (i + 1)) + (r_mse[i] / (i + 1))
105
106 return r_mse
107
108
[docs]
109def mape(y_pred: np.ndarray, y: np.ndarray, epsilon: float = .05) -> float:
110 """
111 Computes the Mean Absolute Percentage Error (MAPE).
112
113 Parameters
114 ----------
115 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
116 Predicted outputs.
117 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
118 Ground truth outputs.
119 epsilon : `float`, optional
120 Small number added to predictions and ground truth to avoid division-by-zero.
121
122 The default is 0.05
123
124 Returns
125 -------
126 `float`
127 MAPE score.
128 """
129 if not isinstance(y_pred, np.ndarray):
130 raise TypeError("'y_pred' must be an instance of 'numpy.ndarray' " +
131 f"but not of '{type(y_pred)}'")
132 if not isinstance(y, np.ndarray):
133 raise TypeError("'y' must be an instance of 'numpy.ndarray' " +
134 f"but not of '{type(y)}'")
135 if not isinstance(epsilon, float):
136 raise TypeError("'epsilon' must be an instance of 'float' " +
137 f"but not of '{type(epsilon)}'")
138 if y_pred.shape != y.shape:
139 raise ValueError(f"Shape mismatch: {y_pred.shape} vs. {y.shape}")
140 if len(y_pred.shape) != 1:
141 raise ValueError("'y_pred' must be a 1d array")
142 if len(y.shape) != 1:
143 raise ValueError("'y' must be a 1d array")
144
145 y_ = y + epsilon
146 y_pred_ = y_pred + epsilon
147 return np.mean(np.abs((y_ - y_pred_) / y_))
148
149
[docs]
150def smape(y_pred: np.ndarray, y: np.ndarray, epsilon: float = .05) -> float:
151 """
152 Computes the Symmetric Mean Absolute Percentage Error (SMAPE).
153
154 Parameters
155 ----------
156 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
157 Predicted outputs.
158 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
159 Ground truth outputs.
160 epsilon : `float`, optional
161 Small number added to predictions and ground truth to avoid division-by-zero.
162
163 The default is 0.05
164
165 Returns
166 -------
167 `float`
168 SMAPE score.
169 """
170 if not isinstance(y_pred, np.ndarray):
171 raise TypeError("'y_pred' must be an instance of 'numpy.ndarray' " +
172 f"but not of '{type(y_pred)}'")
173 if not isinstance(y, np.ndarray):
174 raise TypeError("'y' must be an instance of 'numpy.ndarray' " +
175 f"but not of '{type(y)}'")
176 if not isinstance(epsilon, float):
177 raise TypeError("'epsilon' must be an instance of 'float' " +
178 f"but not of '{type(epsilon)}'")
179 if y_pred.shape != y.shape:
180 raise ValueError(f"Shape mismatch: {y_pred.shape} vs. {y.shape}")
181 if len(y_pred.shape) != 1:
182 raise ValueError("'y_pred' must be a 1d array")
183 if len(y.shape) != 1:
184 raise ValueError("'y' must be a 1d array")
185
186 y_ = y + epsilon
187 y_pred_ = y_pred + epsilon
188 return 2. * np.mean(np.abs(y_ - y_pred_) / (np.abs(y_) + np.abs(y_pred_)))
189
190
[docs]
191def mase(y_pred: np.ndarray, y: np.ndarray, epsilon: float = .05) -> float:
192 """
193 Computes the Mean Absolute Scaled Error (MASE).
194
195 Parameters
196 ----------
197 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
198 Predicted outputs.
199 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
200 Ground truth outputs.
201 epsilon : `float`, optional
202 Small number added to predictions and ground truth to avoid division-by-zero.
203
204 The default is 0.05
205
206 Returns
207 -------
208 `float`
209 MASE score.
210 """
211 if not isinstance(y_pred, np.ndarray):
212 raise TypeError("'y_pred' must be an instance of 'numpy.ndarray' " +
213 f"but not of '{type(y_pred)}'")
214 if not isinstance(y, np.ndarray):
215 raise TypeError("'y' must be an instance of 'numpy.ndarray' " +
216 f"but not of '{type(y)}'")
217 if not isinstance(epsilon, float):
218 raise TypeError("'epsilon' must be an instance of 'float' " +
219 f"but not of '{type(epsilon)}'")
220 if y_pred.shape != y.shape:
221 raise ValueError(f"Shape mismatch: {y_pred.shape} vs. {y.shape}")
222 if len(y_pred.shape) != 1:
223 raise ValueError("'y_pred' must be a 1d array")
224 if len(y.shape) != 1:
225 raise ValueError("'y' must be a 1d array")
226
227 try:
228 y_ = y + epsilon
229 y_pred_ = y_pred + epsilon
230
231 mae = mean_absolute_error(y_, y_pred_)
232 naive_error = np.mean(np.abs(y_[1:] - y_pred_[:-1]))
233 return mae / naive_error
234 except Exception:
235 return None
236
237
[docs]
238def f1_micro_score(y_pred: np.ndarray, y: np.ndarray) -> float:
239 """
240 Computes the F1 score using for a multi-class classification by
241 counting the total true positives, false negatives and false positives.
242
243 Parameters
244 ----------
245 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
246 Predicted labels.
247 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
248 Ground truth labels.
249
250 Returns
251 -------
252 `float`
253 F1 score.
254 """
255 if not isinstance(y_pred, np.ndarray):
256 raise TypeError("'y_pred' must be an instance of 'numpy.ndarray' " +
257 f"but not of '{type(y_pred)}'")
258 if not isinstance(y, np.ndarray):
259 raise TypeError("'y' must be an instance of 'numpy.ndarray' " +
260 f"but not of '{type(y)}'")
261 if y_pred.shape != y.shape:
262 raise ValueError(f"Shape mismatch: {y_pred.shape} vs. {y.shape}")
263
264 return skelarn_f1_scpre(y, y_pred, average="micro")
265
266
[docs]
267def roc_auc_score(y_pred: np.ndarray, y: np.ndarray) -> float:
268 """
269 Computes the Area Under the Curve (AUC) of a classification.
270
271 Parameters
272 ----------
273 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
274 Predicted labels.
275 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
276 Ground truth labels.
277
278 Returns
279 -------
280 `float`
281 ROC AUC score.
282 """
283 if not isinstance(y_pred, np.ndarray):
284 raise TypeError("'y_pred' must be an instance of 'numpy.ndarray' " +
285 f"but not of '{type(y_pred)}'")
286 if not isinstance(y, np.ndarray):
287 raise TypeError("'y' must be an instance of 'numpy.ndarray' " +
288 f"but not of '{type(y)}'")
289 if y_pred.shape != y.shape:
290 raise ValueError(f"Shape mismatch: {y_pred.shape} vs. {y.shape}")
291
292 return skelarn_roc_auc_score(y, y_pred)
293
294
[docs]
295def true_positive_rate(y_pred: np.ndarray, y: np.ndarray) -> float:
296 """
297 Computes the true positive rate (also called sensitivity).
298
299 Parameters
300 ----------
301 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
302 Predicted labels.
303 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
304 Ground truth labels.
305
306 Returns
307 -------
308 `float`
309 True positive rate.
310 """
311 if not isinstance(y_pred, np.ndarray):
312 raise TypeError("'y_pred' must be an instance of 'numpy.ndarray' " +
313 f"but not of '{type(y_pred)}'")
314 if not isinstance(y, np.ndarray):
315 raise TypeError("'y' must be an instance of 'numpy.ndarray' " +
316 f"but not of '{type(y)}'")
317 if y_pred.shape != y.shape:
318 raise ValueError(f"Shape mismatch: {y_pred.shape} vs. {y.shape}")
319 if len(y_pred.shape) != 1:
320 raise ValueError("'y_pred' must be a 1d array")
321 if len(y.shape) != 1:
322 raise ValueError("'y' must be a 1d array")
323 if set(np.unique(y_pred)) != set([0, 1]):
324 raise ValueError("Labels must be either '0' or '1'")
325
326 tp = np.sum((y == 1) & (y_pred == 1))
327 fn = np.sum((y == 1) & (y_pred == 0))
328
329 return tp / (tp + fn)
330
331
[docs]
332def true_negative_rate(y_pred: np.ndarray, y: np.ndarray) -> float:
333 """
334 Computes the true negative rate (also called specificity).
335
336 Parameters
337 ----------
338 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
339 Predicted labels.
340 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
341 Ground truth labels.
342
343 Returns
344 -------
345 `float`
346 True negative rate.
347 """
348 if not isinstance(y_pred, np.ndarray):
349 raise TypeError("'y_pred' must be an instance of 'numpy.ndarray' " +
350 f"but not of '{type(y_pred)}'")
351 if not isinstance(y, np.ndarray):
352 raise TypeError("'y' must be an instance of 'numpy.ndarray' " +
353 f"but not of '{type(y)}'")
354 if y_pred.shape != y.shape:
355 raise ValueError(f"Shape mismatch: {y_pred.shape} vs. {y.shape}")
356 if len(y_pred.shape) > 1:
357 raise ValueError("'y_pred' must be a 1d array")
358 if len(y.shape) > 1:
359 raise ValueError("'y' must be a 1d array")
360 if set(np.unique(y_pred)) != set([0, 1]):
361 raise ValueError("Labels must be either '0' or '1'")
362
363 tn = np.sum((y == 0) & (y_pred == 0))
364 fp = np.sum((y == 0) & (y_pred == 1))
365
366 return tn / (tn + fp)
367
368
[docs]
369def precision_score(y_pred: np.ndarray, y: np.ndarray) -> float:
370 """
371 Computes the precision of a classification.
372
373 Parameters
374 ----------
375 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
376 Predicted labels.
377 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
378 Ground truth labels.
379
380 Returns
381 -------
382 `float`
383 Precision score.
384 """
385 if not isinstance(y_pred, np.ndarray):
386 raise TypeError("'y_pred' must be an instance of 'numpy.ndarray' " +
387 f"but not of '{type(y_pred)}'")
388 if not isinstance(y, np.ndarray):
389 raise TypeError("'y' must be an instance of 'numpy.ndarray' " +
390 f"but not of '{type(y)}'")
391 if y_pred.shape != y.shape:
392 raise ValueError(f"Shape mismatch: {y_pred.shape} vs. {y.shape}")
393 if set(np.unique(y_pred)) != set([0, 1]):
394 raise ValueError("Labels must be either '0' or '1'")
395
396 tp = np.sum([np.all((y[i] == 1) & (y_pred[i] == 1)) for i in range(len(y))])
397 fp = np.sum([np.any((y[i] == 0) & (y_pred[i] == 1)) for i in range(len(y))])
398
399 return tp / (tp + fp)
400
401
[docs]
402def accuracy_score(y_pred: np.ndarray, y: np.ndarray) -> float:
403 """
404 Computes the accuracy of a classification.
405
406 Parameters
407 ----------
408 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
409 Predicted labels.
410 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
411 Ground truth labels.
412
413 Returns
414 -------
415 `float`
416 Accuracy score.
417 """
418 if not isinstance(y_pred, np.ndarray):
419 raise TypeError("'y_pred' must be an instance of 'numpy.ndarray' " +
420 f"but not of '{type(y_pred)}'")
421 if not isinstance(y, np.ndarray):
422 raise TypeError("'y' must be an instance of 'numpy.ndarray' " +
423 f"but not of '{type(y)}'")
424 if y_pred.shape != y.shape:
425 raise ValueError(f"Shape mismatch: {y_pred.shape} vs. {y.shape}")
426
427 tp = np.sum([np.all(y[i] == y_pred[i]) for i in range(len(y))])
428 return tp / len(y)
429
430
[docs]
431def f1_score(y_pred: np.ndarray, y: np.ndarray) -> float:
432 """
433 Computes the F1-score for a binary classification.
434
435 Parameters
436 ----------
437 y_pred : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
438 Predicted labels.
439 y : `numpy.ndarray <https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html>`_
440 Ground truth labels.
441
442 Returns
443 -------
444 `float`
445 F1-score.
446 """
447 if not isinstance(y_pred, np.ndarray):
448 raise TypeError("'y_pred' must be an instance of 'numpy.ndarray' " +
449 f"but not of '{type(y_pred)}'")
450 if not isinstance(y, np.ndarray):
451 raise TypeError("'y' must be an instance of 'numpy.ndarray' " +
452 f"but not of '{type(y)}'")
453 if y_pred.shape != y.shape:
454 raise ValueError(f"Shape mismatch: {y_pred.shape} vs. {y.shape}")
455 if len(y_pred.shape) != 1:
456 raise ValueError("'y_pred' must be a 1d array")
457 if len(y.shape) != 1:
458 raise ValueError("'y' must be a 1d array")
459 if set(np.unique(y_pred)) != set([0, 1]):
460 raise ValueError("Labels must be either '0' or '1'")
461
462 tp = np.sum((y == 1) & (y_pred == 1))
463 fp = np.sum((y == 0) & (y_pred == 1))
464 fn = np.sum((y == 1) & (y_pred == 0))
465
466 return (2. * tp) / (2. * tp + fp + fn)