No-show forecasting notebook¶

# Install the shapely and datarobot libraries
!pip install shapely --quiet
!pip install datarobot --quiet

# First load the needed libraries
import datarobot as dr
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import datetime as datetime
import shapely.wkt
import shapely.geometry
import os
from IPython.display import HTML
import json
import requests
import yaml

%matplotlib inline

# Define some useful functions, primarily used at prediction time

def predict_deployment_expl(deployment, data):
    # Get the DataRobot key, prediction server URL, and deployment ID
    pred_server = deployment.default_prediction_server
    datarobot_key = pred_server['datarobot-key']
    deployment_url = pred_server['url']
    deployment_id = deployment.id
    
    # Set HTTP headers. Match the charset to the contents of the file
    headers = {'Content-Type': 'application/json; charset=UTF-8', 'datarobot-key': datarobot_key}
    
    # Set Prediction Explanations parameters
    params = {
            'maxCodes': 3,
            'thresholdHigh': 0.1,
            'thresholdLow': 0.01,
        }
    
    url = f'{deployment_url}/predApi/v1.0/deployments/{deployment_id}/predictionExplanations'
    
    # Make an API request for predictions
    predictions_response = requests.post(
        url,
        auth=(creds['username'], creds['token']),
        data=data,
        headers=headers,
        params=params
    )

    return predictions_response.json()

# The prediction API returns values in nested JSON objects. The function below flattens the results to tabular format.
def flatten_json(y):
    out = {}

    def flatten(x, name=''):
        if type(x) is dict:
            for a in x:
                flatten(x[a], name + a + '_')
        elif type(x) is list:
            i = 0
            for a in x:
                flatten(a, name + str(i) + '_')
                i += 1
        else:
            out[name[:-1]] = x

    flatten(y)
    return out

def make_prediction(deployment, df_scoring):
    # Convert to JSON format
    data_to_pred = json.dumps(df_scoring.to_dict(orient='records'))
    
    # Get and process predictions with explanations
    preds_raw = predict_deployment_expl(deployment, data_to_pred)
    preds_lst = [flatten_json(row) for row in preds_raw['data']]
    df_preds = pd.DataFrame(preds_lst)
    
    cols_to_rename = {'predictionValues_0_value': 'Prediction', 
                  'predictionExplanations_0_feature': 'Primary Feature',
                  'predictionExplanations_0_featureValue': 'Primary Feature Value',
                  'predictionExplanations_0_qualitativeStrength':'Primary Feature Strength',
                  
                  'predictionExplanations_1_feature': 'Secondary Feature',
                  'predictionExplanations_1_featureValue': 'Secondary Feature Value',
                  'predictionExplanations_1_qualitativeStrength':'Secondary Feature Strength',
                  
                  'predictionExplanations_2_feature': 'Tertiary Feature',
                  'predictionExplanations_2_featureValue': 'Tertiary Feature Value',
                  'predictionExplanations_2_qualitativeStrength':'Tertiary Feature Strength'}

    df_preds = df_preds.rename(columns=cols_to_rename)
    
    return df_preds

def merge_results(df_scoring, df_preds):
    cols_to_add = ['passthroughValues_Appointment ID', 'Prediction', 
                   'Primary Feature Value', 'Primary Feature Strength', 'Primary Feature', 
                   'Secondary Feature Value', 'Secondary Feature Strength', 'Secondary Feature',
                   'Tertiary Feature Value', 'Tertiary Feature Strength', 'Tertiary Feature',
                  ]
    df_scoring = df_scoring.merge(df_preds[cols_to_add], left_on='Appointment ID', right_on='passthroughValues_Appointment ID')
    df_scoring['no_show'] = df_scoring.Prediction.apply(lambda x: True if x > 0.5 else False)
    df_scoring = df_scoring.drop(columns=['passthroughValues_Appointment ID'])
    return df_scoring

def data_prep_sample(sample_df):
    sample_df['Schedule Date'] = pd.to_datetime(sample_df['Schedule Date'])
    sample_df['Appointment Date'] = pd.to_datetime(sample_df['Appointment Date'])

    # Calculate the delta between scheduled day and appointment day. Remove time to return in days only.
    sample_df["Appointment/Schedule Delta"] = (sample_df['Appointment Date'] - sample_df['Schedule Date'].apply(lambda t : t.replace(minute=0, hour=0, second=0, microsecond=0))).apply(lambda t: t.days)

    # Calculate a new column for the distance between the neighborhood and the clinic
    sample_df['Distance'] = df.apply(lambda l: calcDistance(l['Neighborhood'], l['Clinic Location']), axis=1)

    sample_df['Appointment Date'] = sample_df['Appointment Date'].apply(lambda l: str(l))
    sample_df['Schedule Date'] = sample_df['Schedule Date'].apply(lambda l: str(l))
    
    return sample_df

def load_file_and_make_predictions(file_name):
    print("loading file " + file_name + "...")
    df_scoring_csv = pd.read_csv(file_name)
    df_scoring = data_prep_sample(df_scoring_csv)
    print("loading file " + file_name + "...done")

    print("Generating predictions...")
    df_preds = make_prediction(deployment, df_scoring)
    results_df = merge_results(df_scoring, df_preds)
    print("Generating predictions...done")
    
    print("No shows predicted for " + file_name)
    display(results_df.loc[results_df['no_show'] == True])

    print()
    
    return results_df

# Change the scheduled and appointment dates from string to date
df['Schedule Date'] = pd.to_datetime(df['Schedule Date'])
df['Appointment Date'] = pd.to_datetime(df['Appointment Date'])

# Add a new column that calulates the schedule delta between scheduled day and appointment day. Remove time to return in days only.
df["Appointment/Schedule Delta"] = (df['Appointment Date'] - df['Schedule Date'].apply(lambda t : t.replace(minute=0, hour=0, second=0, microsecond=0))).apply(lambda t: t.days)

# Create a map of the neighborhood to the geometry of that neighborhood
geo_df = pd.read_csv('planning_neighborhoods.csv')
geo_df = geo_df.reset_index()
geo = {}

for index, row in geo_df.iterrows():
    geo[row['neighborho']] = row['the_geom']
    
# Create a map of the clinic location to the latitude/longitude of the clinic
clinic_df = pd.read_csv('clinics.csv')
clinic_df = clinic_df.reset_index()
clinic = {}

for index, row in clinic_df.iterrows():
    clinic[row['name']] = [row['lat'], row['long']]

# Calculate the distance between the neighborhood and the clinic
def calcDistance(neigh, loc):
    # Load the geometry (WKT format).
    p = shapely.wkt.loads(geo[neigh])
    lat = clinic[loc][0]
    long = clinic[loc][1]
    # Convert the lat/long into a point.
    point = shapely.geometry.Point(long, lat)
    # Calculate the distance between the centroid of the neighborhood and the point of the clinic
    return p.distance(point)

# Add a new column that is the distance between the neighborhood and the clinic
df['Distance'] = df.apply(lambda l: calcDistance(l['Neighborhood'], l['Clinic Location']), axis=1)

# Create a map of patient ID to historical no show rates
hist_no_show_df = pd.read_csv('no_show_historical.csv')
hist_no_show_df = hist_no_show_df.reset_index()
hist_no_show = {}

for index, row in hist_no_show_df.iterrows():
    hist_no_show[row['Patient ID']] = row['no_show']

# Add a new column that has the historical no show rate for that patient, or 0.0 if there is no data
df['Hist No Show'] = df['Patient ID'].apply(lambda l: hist_no_show.get(l, 0.0))

# Drop the column "Appointment ID" as it will have no meaning in DataRobot
df = df.drop(columns=['Appointment ID'])

# Create the project
ct = datetime.datetime.now()
project_name = f"no_show_enriched_{int(ct.timestamp())}.csv"

project = dataset.create_project(project_name=project_name)

# Create a new Feature List that contains all of the features in the "Informative Features" and then removes some features 
flists = project.get_featurelists()
flist = next(x for x in flists if x.name == "Informative Features")

# Remove the date features from the list, but retain the components created by DataRobot
new_features = [x for x in flist.features if x not in ['Appointment Date','Schedule Date']]

# Create the new Feature List 
fl = project.create_featurelist(name='No Dates', features=new_features)

# Uncomment the next line to print out a link that can be used to access the project from the UI.
#display(HTML(f'You can <a target="_blank" rel="noopener noreferrer" href="https://app.datarobot.com/projects/{project.id}/eda">click into the project</a> and see the new Feature List. '))

# Set the project options--target, partitioning method, and Feature List. Then, start the <a href="https://docs.datarobot.com/en/docs/modeling/reference/model-detail/model-ref.html#quick-autopilot">Quick Autopilot</a>.
group_cv = dr.GroupCV(20, 5, ['Patient ID'])

project.analyze_and_model(
    # Set the target as the no_show column
    target='no_show',
    # Don't limit the worker count
    worker_count = '-1',
    # Group partition on the Patient ID column
    partitioning_method=group_cv,
    # Set the feature list that was created above
    featurelist_id=fl.id
)

# Uncomment the next line to print out a link that can be used to access the project from the UI.
#display(HTML(f'Now you can navigate to the models page of the new project or <a target="_blank" rel="noopener noreferrer" href="https://app.datarobot.com/projects/{project.id}/models">click this link</a>'))

project.wait_for_autopilot(verbosity=0)