Upload Content In Parts Example

Hardware Connectivity
1

If a device content file is larger than 100MB in size, you must upload the file in multiple parts. The documentation on how to do that can be found here: Device HTTP API - Exosite Documentation

Below is a python example of uploading a file larger than 100MB

import os
import json
import requests
import base64
import hashlib
from requests.adapters import HTTPAdapter, Retry

# Documentation: https://docs.exosite.io/device-connectivity/device-http-api/#upload-content-in-parts

# TODO:
# ensure correct product_id, token, and file_type are set

# Assumptions
# - Device is already provisioned
# - Using device token authentication
# - File is larger than 100MB (if smaller, use a single HTTP call: https://docs.exosite.io/device-connectivity/device-http-api/#upload-content)

def main():
    # Need file size for the first POST call to initiate the upload
    # We'll open this file later
    file_name = "bigfile.txt"
    content_length = os.path.getsize(file_name)

    #TODO: Modify these variables
    # Solution/product id of the IoT Connector
    product_id = "<product_id>"
    # The device token (assuming using token authentication method)
    token = "<device_token>" 
    # Change this file_type to match the type (MIME type) of file being uploaded: https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types
    file_type = "text/plain"

    upload_url = f"https://{product_id}.m2.exosite.io/onep:v1/content/"

    # Implement retries incase a request fails
    req_session = requests.Session()
    retries = Retry(total=5, backoff_factor=1, status_forcelist=[ 502, 503, 504 ])
    req_session.mount('https://', HTTPAdapter(max_retries=retries))

    #first POST that initializes the upload
    headers = {
        'Content-Type': 'application/json',
        'X-Exosite-CIK': token
    }
    info = {
            "id": file_name,
            "length": content_length,
            "type": file_type
    }

    r = req_session.post(
        upload_url,
        headers=headers,
        data = json.dumps(info)
        )

    print(json.dumps(info))
    print(r, r.text)

    # Open file and loop on 5MB portions
    range_pointer = 0 # Marker for knowing the beginning of the portion we are sending up
    upload_url = f"https://{product_id}.m2.exosite.io/onep:v1/content/{file_name}"

    # The method for opening the file might need to change based on local OS
    in_file = open(f"./{file_name}", "rb")
    while True:
        # The partial payloads need to be 5MB of data
        data = in_file.read(5*1024*1024)
        data_len = len(data) # Not always 5MB on the last data read

        if data_len == 0:
            break # EOF

        MD5_raw = hashlib.md5(data).digest()
        MD5_base64 = base64.b64encode(MD5_raw).decode("utf-8")

        headers = {
            'X-Exosite-CIK': token,
            'Content-Type': 'application/octet-stream',
            'MD5': MD5_base64,
            'Content-Range': f"bytes {str(range_pointer)}-{str(range_pointer + data_len - 1)}/{str(content_length)}"
        }
        print(json.dumps(headers))

        # Send partial file
        r = req_session.patch(
            upload_url,
            headers=headers,
            data=data)
        print(r, r.text)
        range_pointer = range_pointer + data_len

    in_file.close()

if __name__ == "__main__":
    main()