import fetch from 'node-fetch'; // In browser environments, 'fetch' is global

async function fetchAllPaginatedData(baseUrl, limit = 100) {
  let allData = [];
  let currentPage = 1;
  let hasMorePages = true;

  while (hasMorePages) {
    const url = `${baseUrl}?page=${currentPage}&limit=${limit}`;
    try {
      const response = await fetch(url);
      if (!response.ok) {
        throw new Error(`HTTP error! status: ${response.status}`);
      }
      const data = await response.json();

      // Assuming the API returns data in 'items' and a 'nextPage' flag or similar
      // Adapt these keys to your specific API's response structure
      if (data && Array.isArray(data.items)) {
        allData = allData.concat(data.items);
        // This is a common pattern: API tells if there's a next page
        hasMorePages = data.nextPage === true; 
        currentPage++;
      } else if (data && Array.isArray(data)) {
        // If the API directly returns an array and you determine pagination by empty response
        allData = allData.concat(data);
        hasMorePages = data.length === limit; // Assume no more if less than limit
        currentPage++;
      } else {
        // No recognizable pagination structure, stop.
        console.warn("API response structure not recognized for pagination. Stopping.");
        hasMorePages = false;
      }

      // If the API uses a 'Link' header for pagination, you'd parse that instead
      // const linkHeader = response.headers.get('Link');
      // hasMorePages = linkHeader && linkHeader.includes('rel="next"');

    } catch (error) {
      console.error(`Failed to fetch page ${currentPage}:`, error);
      hasMorePages = false; // Stop on error
    }
  }
  return allData;
}

// Example Usage:
// (async () => {
//   try {
//     // Replace with your actual paginated API endpoint
//     const data = await fetchAllPaginatedData("https://api.example.com/items", 50);
//     console.log(`Fetched ${data.length} items from all pages.`);
//     // console.log("First 5 items:", data.slice(0, 5));
//   } catch (error) {
//     console.error("Error fetching all paginated data:", error);
//   }
// })();