import requests
def fetch_all_data():
    base_url = 'https://datacatalogapi.worldbank.org/dexapps/fone/api/apiservice'
    dataset_id = 'DS04621'
    resource_id = 'RS03878'
    top = 1000
    skip = 0
    all_data = []
    total_count = 0
    page = 1
    while True:
        url = f'{base_url}?datasetId={dataset_id}&resourceId={resource_id}&top={top}&type=json&skip={1000*(page-1)}'
        try:
            response = requests.get(url, timeout=10)
            #print(response)
            response.raise_for_status()  # Raise an exception for 4XX and 5XX status codes
            data = response.json()  # Parse the JSON response

            # Assuming the actual data is in the 'data' key of the response
            if 'data' not in data or not data['data']:  # Exit the loop if no more data is returned
                break

            if page == 1 and 'count' in data:
                total_count = data['count']  # Store the total count from the first response

            all_data.extend(data['data'])  # Add the fetched data to the list
            page += 1  # Increment the page value to get the next set of records
            #print(f'Page: {page}, Status Code: {response.status_code}')
        except requests.RequestException as e:
            print(f'Error: {e}')
            break
    return {'count': total_count, 'data': all_data}
# Fetch all data and print it
all_data = fetch_all_data()
print(all_data)

function Fetch-AllData {
    param (
        [string]$baseUrl
    )
    $allData = @()
    $top = 1000
    $skip = 0
    $hasMoreData = $true
    while ($hasMoreData) {
        $url = "$baseUrl&top=$top&skip=$skip"
        try {
            $response = Invoke-RestMethod -Uri $url -Method Get
            $allData += $response.data

            # Check if we received less data than requested, indicating no more data
            if ($response.data.Count -lt $top) {
                $hasMoreData = $false
            } else {
                $skip += $top
            }
        } catch {
            Write-Error "Error: $_"
            $hasMoreData = $false
        }
    }
    return $allData
}
$baseUrl = 'https://datacatalogapi.worldbank.org/dexapps/fone/api/apiservice?datasetId=DS04621&resourceId=RS03878&type=json'
$allData = Fetch-AllData -baseUrl $baseUrl
# Do something with all the data
$allData | ConvertTo-Json | Write-Output

require 'net/http'
require 'json'
require 'uri'
 
def fetch_all_data(base_url)
  all_data = []
  has_more_data = true
  top = 1000
  skip = 0
  while has_more_data
 
    url = URI("#{base_url}&top=#{top}&skip=#{skip}")
    response = Net::HTTP.get_response(url)
    if response.is_a?(Net::HTTPSuccess)
      begin
        data = JSON.parse(response.body)
        if data.key?('data')
          all_data.concat(data['data'])
          # Check if we received less data than requested, indicating no more data
          if data['data'].length < top
            has_more_data = false
          else
            skip += top
          end
        else
          has_more_data = false
        end
      rescue JSON::ParserError => e
        puts "JSON parsing error: #{e.message}"
        has_more_data = false
      end
    else
      puts "Error: #{response.code} - #{response.message}"
      has_more_data = false
    end
  end
  all_data
rescue StandardError => e
  puts "Error: #{e.message}"
end
base_url = 'https://datacatalogapi.worldbank.org/dexapps/fone/api/apiservice?datasetId=DS04621&resourceId=RS03878&type=json'
all_data = fetch_all_data(base_url)
puts all_data

using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;
class Program
{
    static async Task Main(string[] args)
    {
        string baseUrl = "https://datacatalogapi.worldbank.org/dexapps/fone/api/apiservice?datasetId=DS04621&resourceId=RS03878&type=json";
        var allData = await FetchAllData(baseUrl);
        // Do something with all the data
        Console.WriteLine(allData);
    }
    static async Task<JArray> FetchAllData(string baseUrl)
    {
        var allData = new JArray();
        int top = 1000;
        int skip = 0;
        bool hasMoreData = true;
        using (HttpClient client = new HttpClient())
        {
            while (hasMoreData)
            {
                string url = $"{baseUrl}&top={top}&skip={skip}";
                var response = await client.GetStringAsync(url);
                var data = JArray.Parse(JObject.Parse(response)["data"].ToString());
                allData.Merge(data);
                // Check if we received less data than requested, indicating no more data
                if (data.Count < top)
                {
                    hasMoreData = false;
                }
                else
                {
                    skip += top;
                }
            }
        }
        return allData;
    }
}

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import org.json.JSONArray;
import org.json.JSONObject;
public class Main {
    public static void main(String[] args) throws Exception {
        String baseUrl = "https://datacatalogapi.worldbank.org/dexapps/fone/api/apiservice?datasetId=DS04621&resourceId=RS03878&type=json";
        int top = 1000;
        int skip = 0;
        boolean hasMoreData = true;
        JSONArray allData = new JSONArray();
        while (hasMoreData) {
            String urlWithParams = baseUrl + "&top=" + top + "&skip=" + skip;
            URL url = new URL(urlWithParams);
            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
            connection.setRequestMethod("GET");
            BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
            String line;
            StringBuffer response = new StringBuffer();
            while ((line = reader.readLine()) != null) {
                response.append(line);
            }
            reader.close();
            JSONObject jsonResponse = new JSONObject(response.toString());
            JSONArray data = jsonResponse.getJSONArray("data");
            for (int i = 0; i < data.length(); i++) {
                allData.put(data.getJSONObject(i));
            }
            // Check if we received less data than requested, indicating no more data
            if (data.length() < top) {
                hasMoreData = false;
            } else {
                skip += top;
            }
        }
        // Do something with all the data
        System.out.println(allData.toString());
    }
}

#!/bin/bash
top=1000
skip=0
has_more_data=true
all_data="[]"
while $has_more_data; do
    url="https://datacatalogapi.worldbank.org/dexapps/fone/api/apiservice?datasetId=DS04621&resourceId=RS03878&type=json&top=$top&skip=$skip"
    response=$(curl -s -X GET "$url")
    # Extract data from the response
    data=$(echo "$response" | jq '.data')
    # Append the new data to all_data
    all_data=$(echo "$all_data $data" | jq -s '[.[][]]')
    # Check if we received less data than requested, indicating no more data
    data_length=$(echo "$data" | jq 'length')
    if [[ "$data_length" -lt "$top" ]]; then
        has_more_data=false
    else
        skip=$((skip + top))
    fi
done
# Output all the data
echo "$all_data" | jq

<?php
function fetchAllData($baseUrl) {
    $allData = [];
    $hasMoreData = true;
    $top = 1000;
    $skip = 0;
 
    while ($hasMoreData) {
        $url = "{$baseUrl}&top={$top}&skip={$skip}";
        $curl = curl_init();
        curl_setopt($curl, CURLOPT_URL, $url);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
        $response = curl_exec($curl);
 
        if (curl_errno($curl)) {
            echo 'Curl error: ' . curl_error($curl);
            curl_close($curl);
            break;
        }
 
        curl_close($curl);
        $data = json_decode($response, true);
 
        if (isset($data['data'])) {
            $allData = array_merge($allData, $data['data']);
            // Check if we received less data than requested, indicating no more data
            if (count($data['data']) < $top) {
                $hasMoreData = false;
            } else {
                $skip += $top;
            }
        } else {
            $hasMoreData = false;
        }
    }
 
    return $allData;
}
 
$baseUrl = 'https://datacatalogapi.worldbank.org/dexapps/fone/api/apiservice?datasetId=DS04621&resourceId=RS03878&type=json';
$allData = fetchAllData($baseUrl);
 
print_r($allData);

clear
local base_url "https://datacatalogapi.worldbank.org/dexapps/fone/api/apiservice?datasetId=DS04621&resourceId=RS03878&type=json"
local top 1000
local skip 0
local has_more_data 1
tempfile all_data
save `all_data', emptyok
while `has_more_data' {
    local url "`base_url'&top=`top'&skip=`skip'"
    import delimited "`url'", clear
    // Check if we received less data than requested, indicating no more data
    local n = _N
    if `n' < `top' {
        local has_more_data 0
    } else {
        local skip = `skip' + `top'
    }
    append using `all_data'
    save `all_data', replace
}
use `all_data'