Migrate a model to a new cluster¶

!{sys.executable} -m pip install --upgrade pip
# !{sys.executable} -m pip install "datarobot>=2.28,<2.29"
!{sys.executable} -m pip install datarobot
!{sys.executable} -m pip install datetime requests --upgrade

!pip3 install --upgrade pip
# !pip3 install "datarobot>=2.28,<2.29"
!pip3 install datarobot
!pip3 install datetime requests --upgrade

import sys
import requests
import time
import json
import os
from datetime import date
import datetime
from timeit import default_timer
import datarobot as dr
import urllib.parse
from IPython.display import display, HTML

print("Started: %s" % (str(datetime.datetime.now())))

# Provide the URL protocol, address (IP or FQDN), and path
# Example: source_host = "http://1.2.3.4"
source_host = 'https://source.datarobot.example.com'
# Do not use https://app.datarobot.com, because users do not have access to the "Enable Experimental API access" permission

# Provide an API key from a user with permission from this cluster
source_apikey = ''

# Provide the source project ID
deployment_id = ''

#### Local save file path ####
# Saves to the model directory by default, using the deployment_id as the file name
model_path = "model/%s.mlpkg" % deployment_id

#### Destination Settings ####
# Example: destination_host = "http://4.3.2.1"
destination_host = 'https://destination.datarobot.example.com'

# Provide an API key from the nodes referenced above from a user with the permissions referenced above
destination_apikey = ''

print("DataRobot client version: %s" % dr.__version__)
print("Source url: %s | deployment_id: %s" % (source_host, deployment_id))
print("Output path: %s" % (model_path))
print("Destinastion url: %s" % (destination_host))

# Code block to ensure that the model directory exists
os.makedirs(os.path.dirname(model_path), exist_ok=True)

# Build the headers and provide the token
headers = {}
headers['Authorization'] = 'Bearer {}'.format(source_apikey)
# Optional - helps DataRobot track usage of this sample
headers['User-Agent'] = 'AIA-E2E-MIGRATION-19'

# Create a new session
session = requests.Session()

session.headers.update(headers)

print("Downloading the mlpkg file from: %s" % source_host)

# Download Code
# Makes request to generate an .mlpkg for download on the target server
# Returns a URL in the location attribute in response header or None
def _request_model_package_download(session, host, deployment_id):

    apiEndpoint = urllib.parse.urljoin( host, "/api/v2/deployments/%s/modelPackageFileBuilds/" % deployment_id )
    print("using download apiEndpoint: %s" % apiEndpoint)

    ssl_verify = True if (urllib.parse.urlparse(host)).scheme == 'https' else False

    try:
        r = session.post(apiEndpoint, verify=ssl_verify)
        r.raise_for_status()
        return r.headers.get('Location')
    except requests.exceptions.HTTPError as err:
        print("Error: %s" % err)
        return None


# Downloads an .mlpkg file to the local system from the target server
# Returns the binary data to be downloaded or None
def get_model_package(session, host, deployment_id):
    location = _request_model_package_download(session, host, deployment_id)
    print("using location: %s" % location)
    ssl_verify = True if (urllib.parse.urlparse(host)).scheme == 'https' else False
    attempts = 0
    wait_length = 30
    r = None
    while attempts <= 10:
        try:
            r = session.get(location, verify=ssl_verify)
            r.raise_for_status()
            print(r.json())
            print("sleeping %s seconds" % wait_length)
            time.sleep(wait_length)
            attempts += 1
        except ValueError:
            print("looks like no json, time to download")
            return r
        except:
            attempts += 1
            print("exception, sleeping for 60 seconds")
            time.sleep(60)
    print("Number of check attempts exceeded. please check the target instance to see if the package is still being assembled or not")
    return None

start = default_timer()

output = get_model_package(session, source_host,  deployment_id)

# if output is None:
#     print("download failed")

print("Saving data to: %s" % model_path)

with open(model_path,'wb') as f:
    f.write(output.content)

print('%s took %s seconds to download %s megs' % (model_path, default_timer() - start, str(round(os.path.getsize(model_path) / (1024 * 1024), 2))))

headers = {}
headers['Authorization'] = 'Bearer {}'.format(destination_apikey)
# Optional - helps DataRobot track usage of this sample
headers['User-Agent'] = 'AIA-E2E-MIGRATION-19'

session = requests.Session()
session.headers.update(headers)

model_name = ""

# Upload code
# Makes a request to upload the .mlpkg file to the target server
# Returns a URL in the location attribute of the response header or None
def _request_package_upload(session, host, fileLocation):

    apiEndpoint = urllib.parse.urljoin( host, "/api/v2/modelPackages/fromFile/" )
    print("using upload apiEndpoint: %s" % apiEndpoint)

    ssl_verify = True if (urllib.parse.urlparse(host)).scheme == 'https' else False

    f = {'file': open(fileLocation, 'rb')}

    try: 
        r = session.post(apiEndpoint, files=f, verify=ssl_verify)
        r.raise_for_status()
        return r.headers.get('Location')
    except requests.exceptions.HTTPError as err:
        print("ERROR: %s" % err)
        return None

# Uploads the .mlpkg file to the target server
# Returns the ID of the new model package or None
def upload_model_package(session, host, fileLocation):
    location = _request_package_upload(session, host, fileLocation)
    print("Location: %s" % location)
    ssl_verify = True if (urllib.parse.urlparse(host)).scheme == 'https' else False

    attempts = 0
    wait_length = 25

    while attempts < 10:
        try:
            r = session.get(location, verify=ssl_verify)
            r.raise_for_status()
            data = r.json()
            # Check if you get a status or if it's redirected to the package object
            if data.get('status') is not None:
                print(data)
            else:
                print("Model Package Uploaded")
                return data.get('id'), data.get('importance'), data.get('name')
            attempts += 1
            print("sleeping %s seconds" % wait_length)
            time.sleep(wait_length)
        except:
            attempts += 1
            print("exception, sleeping 60")
            time.sleep(60)

    print("ERROR: Number of check attempts exceeded. please check the target instance to see if there are errors")
    return None    

# Upload the .mlpkg
start = default_timer()
print("Uploading file: %s to: %s" % (model_path, destination_host) )

destination_model_id, destination_model_importance, destination_model_name = upload_model_package(session, destination_host, model_path)

if destination_model_id is None:
    print("upload failed")
else:

    link =  urllib.parse.urljoin(destination_host, "/model-registry/model-packages/%s" % destination_model_id)
    print("Upload took %s seconds" % ( default_timer() - start ))

dpeEndpoint = "%s/api/v2/predictionServers/" % (destination_host)
prediction_environment_id = None
prediction_environment_url = None

ssl_verify = True if (urllib.parse.urlparse(destination_host)).scheme == 'https' else False

print("finding dpe with: %s" % dpeEndpoint)
try:
    r = session.get(dpeEndpoint, verify=ssl_verify)
    r.raise_for_status()

    data = json.loads(r.text)

    prediction_environment_id = data['data'][data['count']-1]['id']
    prediction_environment_url = data['data'][data['count']-1]['url']

except requests.exceptions.HTTPError as err:
    print("Error: %s" % err)
    raise Exception("Error: %s" % err)

## Debug
# print("data: %s" % data )

print("Found DPE id: %s | url: %s" % ( prediction_environment_id, prediction_environment_url )  )

# Returns Deployment ID or None
def deploy_model(session, pid, mid, imp):
    apiEndpoint = "%s/api/v2/deployments/fromModelPackage/" % destination_host
    print("deploy from: %s" % apiEndpoint)
    ssl_verify = True if (urllib.parse.urlparse(destination_host)).scheme == 'https' else False

    body_payload = {
        "label": "%s" % (destination_model_name),
        "description": "Cloned from: %s" % (urllib.parse.urlparse(source_host).netloc),
        "modelPackageId": mid,
        "importance": imp
    }
    print("deployment settings: %s" % body_payload)

    try:
        r = session.post(
            apiEndpoint,
            data=json.dumps(body_payload),
            headers ={'Content-Type': 'application/json', 'Accept': 'application/json'},
            verify=ssl_verify
        )
        r.raise_for_status()
        return r.text
    except requests.exceptions.HTTPError as err:
        print("ERROR: %s" % err)
        print(r.text)
        print(r.headers)
        return None

start = default_timer()

if (destination_model_importance is None):
    destination_model_importance = "LOW"

output = deploy_model(session, prediction_environment_id, destination_model_id, destination_model_importance )


print("Deplyment of: %s took: %s seconds" % ( output, default_timer() - start))

# # Debug command
# destination_model_id = "foo"
app_node = (urllib.parse.urlparse(destination_host)).netloc

print("# Copy the commands below and paste them to the")
print("# ssh command prompt on: %s" % app_node)
print("")
print("sudo su - datarobot")
print("docker exec -it app /entrypoint /bin/bash -c \"python3 support/upgrade_model_packages.py --save %s\"" % destination_model_id )