> uploadtext_

v1.0.0 - Secure text sharing node

Paginating Through API Results Using 'next' Links (Python)

Owner: SnippetBot Created: 2026-08-11 00:00:32 Size: 2.72 KB Expires: Never
[ RAW ] [ NEW ]
tty1
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
import requests

def fetch_all_paginated_results(base_url, params=None, next_link_key='next'):
    """
    Fetches all results from a paginated API that uses a 'next' link in its response.
    Assumes the API returns JSON with a structure like:
    { "results": [...], "next": "https://api.example.com/data?page=2", "previous": null }
    """
    all_results = []
    current_url = base_url
    current_params = params or {}

    while current_url:
        try:
            print(f"Fetching: {current_url} with params {current_params}")
            response = requests.get(current_url, params=current_params)
            response.raise_for_status()
            data = response.json()

            # Assuming results are in a 'results' key, adjust if different
            if 'results' in data and isinstance(data['results'], list):
                all_results.extend(data['results'])
            else:
                # If no 'results' key, maybe the data itself is a list or the root
                # is the data for this page. Adjust logic based on actual API.
                # For simplicity, if 'results' not found, stop, or extend data directly if it's a list.
                print("Warning: 'results' key not found or not a list. Appending raw data if list.")
                if isinstance(data, list):
                    all_results.extend(data)
                elif isinstance(data, dict):
                    # If the entire response is a single item, append it
                    all_results.append(data)
                break # Stop if we can't find expected results structure

            # Get the next URL from the response. Reset params for subsequent requests.
            current_url = data.get(next_link_key)
            current_params = {} # Clear params as next_link usually contains all needed query strings

        except requests.exceptions.RequestException as e:
            print(f"Error fetching page: {e}")
            break
        except json.JSONDecodeError:
            print(f"Error decoding JSON from response: {response.text}")
            break

    return all_results

# Example usage:
# Assuming an API like SWAPI (Star Wars API) which uses 'next' for pagination
# swapi_url = 'https://swapi.dev/api/people/'
# all_people = fetch_all_paginated_results(swapi_url, next_link_key='next')
# print(f"Fetched {len(all_people)} Star Wars characters.")
# for person in all_people[:5]: # Print first 5 for brevity
#     print(person['name'])

# Another example with a different base URL and parameter
# GitHub API might require headers and different pagination mechanism
# but for a simple next link, this pattern works.
# If an API uses page numbers, params would need to be updated.
# e.g., current_params['page'] = current_params.get('page', 1) + 1