import requests def fetch_all_paginated_results(base_url, params=None, next_link_key='next'): """ Fetches all results from a paginated API that uses a 'next' link in its response. Assumes the API returns JSON with a structure like: { "results": [...], "next": "https://api.example.com/data?page=2", "previous": null } """ all_results = [] current_url = base_url current_params = params or {} while current_url: try: print(f"Fetching: {current_url} with params {current_params}") response = requests.get(current_url, params=current_params) response.raise_for_status() data = response.json() # Assuming results are in a 'results' key, adjust if different if 'results' in data and isinstance(data['results'], list): all_results.extend(data['results']) else: # If no 'results' key, maybe the data itself is a list or the root # is the data for this page. Adjust logic based on actual API. # For simplicity, if 'results' not found, stop, or extend data directly if it's a list. print("Warning: 'results' key not found or not a list. Appending raw data if list.") if isinstance(data, list): all_results.extend(data) elif isinstance(data, dict): # If the entire response is a single item, append it all_results.append(data) break # Stop if we can't find expected results structure # Get the next URL from the response. Reset params for subsequent requests. current_url = data.get(next_link_key) current_params = {} # Clear params as next_link usually contains all needed query strings except requests.exceptions.RequestException as e: print(f"Error fetching page: {e}") break except json.JSONDecodeError: print(f"Error decoding JSON from response: {response.text}") break return all_results # Example usage: # Assuming an API like SWAPI (Star Wars API) which uses 'next' for pagination # swapi_url = 'https://swapi.dev/api/people/' # all_people = fetch_all_paginated_results(swapi_url, next_link_key='next') # print(f"Fetched {len(all_people)} Star Wars characters.") # for person in all_people[:5]: # Print first 5 for brevity # print(person['name']) # Another example with a different base URL and parameter # GitHub API might require headers and different pagination mechanism # but for a simple next link, this pattern works. # If an API uses page numbers, params would need to be updated. # e.g., current_params['page'] = current_params.get('page', 1) + 1