chrismbryant / classification-curves Goto Github PK
View Code? Open in Web Editor NEWHome Page: https://chrismbryant.github.io/classification-curves
License: MIT License
Home Page: https://chrismbryant.github.io/classification-curves
License: MIT License
Metrics are currently computed one threshold value at a time and vectorized across bootstraps. Ideally computation would be vectorized across both, but most systems run into memory issues that prohibit this operation. Instead, we should use multiprocessing
to parallelize the for-loop over thresholds across multiple cores (https://www.delftstack.com/howto/python/parallel-for-loops-python/).
Demonstrate functionality with an example model workflow
from typing import Optional, List
import pandas as pd
from ??? import BinomialCI
import sklearn.metrics
import matplotlib as mpl
import numpy as np
import matplotlib.pyplot as plt
print("Imported functions:")
print(" >> assess_prob_calibration(...)")
def assess_prob_calibration(
pred_df: pd.DataFrame,
probability_col: str = "prob",
label_col: str = "label",
) -> pd.DataFrame:
pred_df["quantile"] = pred_df[probability_col].rank() / len(pred_df)
pred_df["quantile_bucket"] = pred_df["quantile"].round(2)
pred_df["score_bucket"] = pred_df[probability_col].round(2)
score_prob_calibration = pred_df.groupby("score_bucket").agg(
avg_prediction=(probability_col, "mean"),
num_actual_pos=(label_col, "sum"),
num_examples=(label_col, "count"),
).reset_index()
quantile_prob_calibration = pred_df.groupby("quantile_bucket").agg(
avg_prediction=(probability_col, "mean"),
num_actual_pos=(label_col, "sum"),
num_examples=(label_col, "count"),
).reset_index()
prob_calibration = pd.concat([score_prob_calibration, quantile_prob_calibration])
prob_calibration["proportion"], prob_calibration["lower"], prob_calibration["upper"] = (
BinomialCI().get_ci(
prob_calibration["num_actual_pos"],
prob_calibration["num_examples"],
)
)
return prob_calibration.sort_values("avg_prediction")
print(" >> plot_probability_calibration(...)")
def plot_probability_calibration(
prob_calibration: pd.DataFrame,
plot_confidence_band: bool = False,
dpi: Optional[int] = None,
color: str = "tab:blue",
title: str = "Probability Calibration Curve",
x_label: str = "Mean Predicted Value",
y_label: str = "Proportion Labeled Positive",
return_fig: bool = False,
) -> None:
fig = plt.figure(figsize = (10, 7), dpi = dpi)
ax = fig.add_subplot(1, 1, 1, aspect = "equal")
ax.grid(True)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
# Add line of perfect calibration
ax.plot([0, 1], [0, 1], "k-")
if plot_confidence_band:
# Shade confidence band
ax.fill_between(
x = prob_calibration["avg_prediction"],
y1 = prob_calibration["lower"],
y2 = prob_calibration["upper"],
alpha = 0.3,
color = color)
ax.plot(
prob_calibration["avg_prediction"],
prob_calibration["proportion"],
color = color)
else:
# Plot points
ax.scatter(
x = prob_calibration["avg_prediction"],
y = prob_calibration["proportion"],
s = 30,
color = color,
marker = ".",
zorder = 100)
# Plot error bars on points
ax.errorbar(
x = prob_calibration["avg_prediction"],
y = prob_calibration["proportion"],
yerr = [
prob_calibration["proportion"] - prob_calibration["lower"],
prob_calibration["upper"] - prob_calibration["proportion"]],
alpha = 0.6,
color = color,
fmt = ".",
markersize = 0,
zorder = 100)
# Set labels
ax.set_xlabel(x_label)
ax.set_ylabel(y_label)
ax.set_title(title)
if return_fig:
return fig, ax
print(" >> plot_predictions(...)")
def plot_predictions(
df: pd.DataFrame,
pred_col: str = "pred",
label_col: str = "label",
dpi: Optional[int] = None,
color: str = "tab:blue",
title: str = "Quantity Predictions",
x_label: str = "Predicted Quantity",
y_label: str = "Actual Quantity",
x_rng: Optional[List[int]] = None,
x_jitter: float = 0,
y_jitter: float = 0,
log_scale: bool = False,
alpha: float = 0.5,
size: float = 20,
vmax: float = 10,
scatter_density: bool = False,
cmap: str = "viridis",
return_fig: bool = False,
) -> None:
fig = plt.figure(figsize=(10, 7), dpi=dpi)
ax = fig.add_subplot(1, 1, 1, aspect="equal")
ax.grid(True, alpha=0.5)
# Gaussian jitter
x_jitter = np.random.normal(0, x_jitter, len(df))
y_jitter = np.random.normal(0, y_jitter, len(df))
if log_scale:
x = df[pred_col] * (1 + x_jitter)
y = df[label_col] * (1 + y_jitter)
else:
x = df[pred_col] + x_jitter
y = df[label_col] + y_jitter
if scatter_density:
import mpl_scatter_density
norm = mpl.colors.Normalize(0, vmax)
ax = fig.add_subplot(1, 1, 1, projection="scatter_density")
ax.scatter_density(x, y, cmap=cmap, norm=norm, dpi=dpi)
else:
ax.scatter(
x = x,
y = y,
s = size,
color = color,
marker = ".",
ec="none",
alpha=alpha
)
ax.set_xlabel(x_label)
ax.set_ylabel(y_label)
ax.set_title(title)
ax.set_aspect("equal")
if x_rng is not None:
ax.set_xlim(x_rng[0], x_rng[1])
ax.set_ylim(x_rng[0], x_rng[1])
# Add line of perfect calibration
ax.plot(
[x_rng[0], x_rng[1]],
[x_rng[0], x_rng[1]],
"k-")
# Include R2, RMSE, and QI as text in plot
df["lit_0"] = 0
r2 = sklearn.metrics.r2_score(df[label_col], df[pred_col])
rmse = np.sqrt(sklearn.metrics.mean_squared_error(df[label_col], df[pred_col]))
rmsle = np.sqrt(sklearn.metrics.mean_squared_log_error(
df[[label_col, "lit_0"]].max(axis=1),
df[[pred_col, "lit_0"]].max(axis=1),
))
qi = df[pred_col].sum() / df[label_col].sum()
ax.text(
x=0.05,
y=0.95,
s=f" $R^2$: {r2:.3f}\n RMSE: {rmse:.3f}\nRMSLE: {rmsle:.3f}\n QI: {qi:.3f}",
horizontalalignment="left",
verticalalignment="top",
transform=ax.transAxes,
fontsize=12
)
if log_scale:
ax.set_yscale("log")
ax.set_xscale("log")
if return_fig:
return fig, ax
Add functionality to support overlaying multiple plots to compare the performance between models.
Support both matched and unmatched samples. (Matched = same group of examples run through 2 different models. Unmatched = one group of examples run through one model, and another [hopefully identically distributed] group run through another model. These have 2 different statistical implications)
This argument currently doesn't work. Should cause input DF to be sampled when specified
Implement a standard for versioning the repo
Currently, builds need to be published manually to PyPI with poetry build
--> poetry publish
. Dev builds should publish to PyPI after every merge to main (and/or every tag release).
Matplotlib made a backwards incompatible change which breaks colorbar usage.
ValueError: Unable to determine Axes to steal space for Colorbar. Either provide the *cax* argument to use as the Axes for the Colorbar, provide the *ax* argument to steal space from it, or add *mappable* to an Axes.
Add an option to plot the confidence band in addition to or instead of the bootstrap examples. Do this by sampling some number of thresholds to compute confidence ellipses for, then connect adjacent threshold ellipses through line segments along their joint convex hull.
Use an argument like:
show_confidence_band: Union[bool, float, List[float]] = False
If True, show only the 95% band. If a float, show the band specified by the float value. If a list of floats, show all the listed bands overlayed with some transparency. Also provide some options for adjusting the transparency and color.
Add support for plotting N one-vs-all curves for a classification problem with N classes. This is useful if some classes are harder to classify than others. When the classes are balanced, the PR plots all converge to the same point at the bottom right. This balancing can be enforced by default using the imbalance multiplier.
Demo on MNIST.
Add bibtex instructions to readme if people want to cite it.
A declarative, efficient, and flexible JavaScript library for building user interfaces.
๐ Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
An Open Source Machine Learning Framework for Everyone
The Web framework for perfectionists with deadlines.
A PHP framework for web artisans
Bring data to life with SVG, Canvas and HTML. ๐๐๐
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
Some thing interesting about web. New door for the world.
A server is a program made to process requests and deliver data to clients.
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Some thing interesting about visualization, use data art
Some thing interesting about game, make everyone happy.
We are working to build community through open source technology. NB: members must have two-factor auth.
Open source projects and samples from Microsoft.
Google โค๏ธ Open Source for everyone.
Alibaba Open Source for everyone
Data-Driven Documents codes.
China tencent open source team.