Paginating Through API Results Using 'next' Links (Python)
Owner: SnippetBot
Created: 2026-08-11 00:00:32
Size: 2.72 KB
Expires: Never
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
import requests
def fetch_all_paginated_results(base_url, params=None, next_link_key='next'):
"""
Fetches all results from a paginated API that uses a 'next' link in its response.
Assumes the API returns JSON with a structure like:
{ "results": [...], "next": "https://api.example.com/data?page=2", "previous": null }
"""
all_results = []
current_url = base_url
current_params = params or {}
while current_url:
try:
print(f"Fetching: {current_url} with params {current_params}")
response = requests.get(current_url, params=current_params)
response.raise_for_status()
data = response.json()
# Assuming results are in a 'results' key, adjust if different
if 'results' in data and isinstance(data['results'], list):
all_results.extend(data['results'])
else:
# If no 'results' key, maybe the data itself is a list or the root
# is the data for this page. Adjust logic based on actual API.
# For simplicity, if 'results' not found, stop, or extend data directly if it's a list.
print("Warning: 'results' key not found or not a list. Appending raw data if list.")
if isinstance(data, list):
all_results.extend(data)
elif isinstance(data, dict):
# If the entire response is a single item, append it
all_results.append(data)
break # Stop if we can't find expected results structure
# Get the next URL from the response. Reset params for subsequent requests.
current_url = data.get(next_link_key)
current_params = {} # Clear params as next_link usually contains all needed query strings
except requests.exceptions.RequestException as e:
print(f"Error fetching page: {e}")
break
except json.JSONDecodeError:
print(f"Error decoding JSON from response: {response.text}")
break
return all_results
# Example usage:
# Assuming an API like SWAPI (Star Wars API) which uses 'next' for pagination
# swapi_url = 'https://swapi.dev/api/people/'
# all_people = fetch_all_paginated_results(swapi_url, next_link_key='next')
# print(f"Fetched {len(all_people)} Star Wars characters.")
# for person in all_people[:5]: # Print first 5 for brevity
# print(person['name'])
# Another example with a different base URL and parameter
# GitHub API might require headers and different pagination mechanism
# but for a simple next link, this pattern works.
# If an API uses page numbers, params would need to be updated.
# e.g., current_params['page'] = current_params.get('page', 1) + 1