FAQ: Use pagination in SDK

Some customers are curious about how to correctly use pagination in the client SDK. Unfortunately, Rockset doesn’t have any new API wrappers implemented yet, but you could check out the below example written in python to leverage REST API for pagination, hope this helps!

import os, requests, json, sys
from dotenv import load_dotenv

load_dotenv()
apiServer = os.getenv(‘ROCKSET_API_SERVER’)
apiKey = os.getenv(‘ROCKSET_API_KEY’)

baseURL = ‘https://’ + apiServer + ‘/v1/orgs/self’
headers={‘Authorization’: 'ApiKey ’ + apiKey , ‘Content-Type’: ‘application/json’}

totalLimit = 990
initialLimit = 100000
pageLimit = 100000

payload = {}
sql = {}
sql[‘query’] = “SELECT * FROM _events’”
sql[‘paginate’] = True
sql[‘initial_paginate_response_doc_count’] = initialLimit
payload[‘sql’] = sql

r = requests.post(
baseURL + ‘/queries’,
json=payload,
headers={‘Authorization’: 'ApiKey ’ + apiKey})

if r.status_code != 200:
print(f’Failed to execute query. Code: {r.status_code}. {r.reason}. {r.text}’)
sys.exit(0)

result = r.json()
queryId = result[‘query_id’]
startCursor = result[‘pagination’][‘start_cursor’]
nextCursor = result[‘pagination’][‘next_cursor’]
totalResults = result[‘results_total_doc_count’]
pagination = result[‘pagination’]
currentCount = pagination[‘current_page_doc_count’]

print(f’Retrieved initial {currentCount} docs. Total doc count: {totalResults}.’)
print(f’ query_id: {queryId} start_cursor: {startCursor} next_cursor: {nextCursor}’ )

page = 0
while nextCursor is not None:
page += 1

r = requests.get(
baseURL + ‘/queries/’ + queryId + ‘/pages/’ + nextCursor + ‘?docs=’ + str(pageLimit),
headers=headers)

result = r.json()
pagination = result[‘pagination’]
currentPageDocCount = pagination[‘current_page_doc_count’] # This will be fewer than requested if there are no more results.
nextCursorOffset = pagination[‘next_cursor_offset’] # This number is the number of documents before the current page.
nextCursor = pagination[‘next_cursor’] # This value will be null if there are no more results.

print(f’Page: {page} current_page_doc_count: {currentPageDocCount} next_cursor: {nextCursor}’ )

1 Like