Python modeling workflow overview¶

# To view target distribution:
df_target_summary = (
    pd.DataFrame(df["SAR"].value_counts())
    .reset_index()
    .rename(columns={"index": "SAR", "SAR": "Count"})
)
ax = sns.barplot(x="SAR", y="Count", data=df_target_summary)

for index, row in df_target_summary.iterrows():
    ax.text(row.SAR, row.Count, round(row.Count, 2), color="black", ha="center")

plt.show()

def get_top_of_leaderboard(project, verbose=True):
    # A helper method to assemble a dataframe with Leaderboard results and print a summary:
    leaderboard = []
    for m in project.get_models():
        leaderboard.append(
            [
                m.blueprint_id,
                m.featurelist.id,
                m.id,
                m.model_type,
                m.sample_pct,
                m.metrics["AUC"]["validation"],
                m.metrics["AUC"]["crossValidation"],
            ]
        )
    leaderboard_df = pd.DataFrame(
        columns=[
            "bp_id",
            "featurelist",
            "model_id",
            "model",
            "pct",
            "validation",
            "cross_validation",
        ],
        data=leaderboard,
    )

    if verbose == True:
        # Print a Leaderboard summary:
        print("Unique blueprints tested: " + str(len(leaderboard_df["bp_id"].unique())))
        print("Feature lists tested: " + str(len(leaderboard_df["featurelist"].unique())))
        print("Models trained: " + str(len(leaderboard_df)))
        print("Blueprints in the project repository: " + str(len(project.get_blueprints())))

        # Print the essential information for top models, sorted by accuracy from validation data:
        print("\n\nTop models on the Leaderboard:")
        leaderboard_top = (
            leaderboard_df[leaderboard_df["pct"] == 64]
            .sort_values(by="cross_validation", ascending=False)
            .head()
            .reset_index(drop=True)
        )
        display(leaderboard_top.drop(columns=["bp_id", "featurelist"], inplace=False))

        # Show blueprints of top models:
        for index, m in leaderboard_top.iterrows():
            Visualize.show_dr_blueprint(dr.Blueprint.get(project.id, m["bp_id"]))

    return leaderboard_top


leaderboard_top = get_top_of_leaderboard(project)

pdm3 = w.Tasks.PDM3(w.TaskInputs.CAT)
pdm3.set_task_parameters(cm=50000, sc=10)

ndc = w.Tasks.NDC(w.TaskInputs.NUM)
rdt5 = w.Tasks.RDT5(ndc)

ptm3 = w.Tasks.PTM3(w.TaskInputs.TXT)
ptm3.set_task_parameters(d2=0.2, mxf=20000, d1=5, n="l2", id=True)

kerasc = w.Tasks.KERASC(rdt5, pdm3, ptm3)
kerasc.set_task_parameters(
    always_use_test_set=1,
    epochs=4,
    hidden_batch_norm=1,
    hidden_units="list(64)",
    hidden_use_bias=0,
    learning_rate=0.03,
    use_training_schedule=1,
)

# Check task documentation:
# kerasc.documentation()

kerasc_blueprint = w.BlueprintGraph(kerasc, name="A Custom Keras BP (1 Layer: 64 Units)").save()
kerasc_blueprint.show()
kerasc_blueprint.train(project_id=project.id, sample_pct=64)

# View the top models on the Leaderboard
leaderboard_top = get_top_of_leaderboard(project)

roc = top_model.get_roc_curve("validation")
df_roc = pd.DataFrame(roc.roc_points)
dr_dark_blue = "#08233F"
dr_roc_green = "#03c75f"
white = "#ffffff"

fig = plt.figure(figsize=(8, 8))
axes = fig.add_subplot(1, 1, 1, facecolor=dr_dark_blue)

plt.scatter(df_roc.false_positive_rate, df_roc.true_positive_rate, color=dr_roc_green)
plt.plot(df_roc.false_positive_rate, df_roc.true_positive_rate, color=dr_roc_green)
plt.plot([0, 1], [0, 1], color=white, alpha=0.25)
plt.title("ROC curve")
plt.xlabel("False Positive Rate (Fallout)")
plt.xlim([0, 1])
plt.ylabel("True Positive Rate (Sensitivity)")
plt.ylim([0, 1])
plt.show()

max_num_features = 15

# Retrieve Feature Impact
feature_impacts = top_model.get_or_request_feature_impact()

# Plot permutation-based Feature Impact
feature_impacts.sort(key=lambda x: x["impactNormalized"], reverse=True)
FeatureImpactDF = pd.DataFrame(
    [
        {"Impact Normalized": f["impactNormalized"], "Feature Name": f["featureName"]}
        for f in feature_impacts[:max_num_features]
    ]
)
FeatureImpactDF["X axis"] = FeatureImpactDF.index
g = sns.lmplot(x="Impact Normalized", y="X axis", data=FeatureImpactDF, fit_reg=False)
sns.barplot(y=FeatureImpactDF["Feature Name"], x=FeatureImpactDF["Impact Normalized"])

feature_effects = top_model.get_or_request_feature_effect(source="validation")
max_features = 5

for f in feature_effects.feature_effects[:max_features]:
    plt.figure(figsize=(9, 6))
    d = pd.DataFrame(f["partial_dependence"]["data"])
    if f["feature_type"] == "numeric":
        d = d[d["label"] != "nan"]
        d["label"] = pd.to_numeric(d["label"])
        sns.lineplot(x="label", y="dependence", data=d).set_title(
            f["feature_name"] + ": importance=" + str(round(f["feature_impact_score"], 2))
        )
    else:
        sns.scatterplot(x="label", y="dependence", data=d).set_title(
            f["feature_name"] + ": importance=" + str(round(f["feature_impact_score"], 2))
        )

# Use training data to test how the model makes predictions
test_data = df.head(50)

dataset_from_file = project.upload_dataset(test_data)
predict_job_1 = top_model.request_predictions(dataset_from_file.id)

predictions = predict_job_1.get_result_when_complete()
display(predictions.head())