Code sample for Pagination in Python

Here’s some sample code on how to implement pagination in Python.

The script takes in the information from a user based on their API Key and the Rockset API server that is saved in a dotenv file. From this, we make a POST request with that information and a payload. The payload includes a SQL query that gets the results for _events:

SELECT * FROM _events

Requirements before running the script

import os, requests, json, sys
from dotenv import load_dotenv


load_dotenv()
apiServer = os.getenv('ROCKSET_API_SERVER')
apiKey = os.getenv('ROCKSET_API_KEY')

baseURL = 'https://' + apiServer + '/v1/orgs/self'
headers={'Authorization': 'ApiKey ' + apiKey , 'Content-Type': 'application/json'}

totalLimit = 990
initialLimit = 100
pageLimit = 100

payload = {}
sql = {}
sql['query'] = "SELECT * FROM _events"
sql['paginate'] = True
sql['initial_paginate_response_doc_count'] = initialLimit
payload['sql'] = sql

r = requests.post(
  baseURL + '/queries',
  json=payload,
  headers={'Authorization': 'ApiKey ' + apiKey})


if r.status_code != 200:
  print(f'Failed to execute query. Code: {r.status_code}. {r.reason}. {r.text}')
  sys.exit(0)

result = r.json()

queryId = result['query_id']
startCursor = result['pagination']['start_cursor']
nextCursor = result['pagination']['next_cursor']
totalResults = result['results_total_doc_count']
pagination = result['pagination']
currentCount = pagination['current_page_doc_count']

print(f'Page 1 {currentCount} docs. Total doc count: {totalResults}.')
print(f'   query_id: {queryId} start_cursor: {startCursor} next_cursor: {nextCursor}' )

page = 1
while nextCursor is not None:
  page += 1

  r = requests.get(
    baseURL + '/queries/' + queryId + '/pages/' + nextCursor + '?docs=' + str(pageLimit),
    headers=headers)

  result = r.json()
  pagination = result['pagination']
  # the actual data will be in result['results']
  currentPageDocCount = pagination['current_page_doc_count'] # This will be fewer than requested if there are no more results.
  nextCursorOffset = pagination['next_cursor_offset'] # This number is the number of documents before the current page.
#  startCursor = pagination['start_cursor']
  nextCursor = pagination['next_cursor'] # This value will be null if there are no more results.

  print(f'Page: {page} current_page_doc_count: {currentPageDocCount}  next_cursor: {nextCursor}' )