API Data Ingestion: REST APIs and JSON Parsing

import json
import urllib.request
import urllib.parse
import ssl

def manual_get_request(url, headers=None):
    """Make a simple HTTP GET request using built-in libraries"""
    request = urllib.request.Request(url)
    if headers:
        for key, value in headers.items():
            request.add_header(key, value)
    
    # Create SSL context that allows us to connect to HTTPS
    context = ssl.create_default_context()
    
    with urllib.request.urlopen(request, context=context) as response:
        data = response.read().decode('utf-8')
        return {
            'status': response.status,
            'headers': dict(response.headers),
            'body': data,
            'json': json.loads(data) if data else None
        }

def simple_json_parser(text):
    """Simple JSON parser using built-in library"""
    return json.loads(text)

def flatten_json(nested_dict, prefix=''):
    """Flatten nested JSON structure for DataFrame conversion"""
    items = []
    for key, value in nested_dict.items():
        new_key = prefix + '.' + key if prefix else key
        if isinstance(value, dict):
            items.extend(flatten_json(value, new_key).items())
        else:
            items.append((new_key, value))
    return dict(items)

# Example JSON handling
sample_json = '{"name": "Alice", "age": 30, "address": {"city": "NYC", "zip": "10001"}}'
parsed = simple_json_parser(sample_json)
flattened = flatten_json(parsed)
print(flattened)

import requests
import pandas as pd
import time

# Basic GET request
# response = requests.get('https://api.example.com/data')
# data = response.json()

# Request with headers and parameters
headers = {'Authorization': 'Bearer YOUR_API_TOKEN', 'Accept': 'application/json'}
params = {'page': 1, 'limit': 100, 'sort': 'created_at'}

# response = requests.get('https://api.example.com/items', headers=headers, params=params)

# POST request with JSON payload
payload = {'name': 'Alice', 'email': 'alice@example.com'}
# response = requests.post('https://api.example.com/users', json=payload, headers=headers)

def fetch_with_retry(url, max_retries=3, backoff=1.0):
    """Fetch URL with exponential backoff retry logic"""
    for attempt in range(max_retries + 1):
        try:
            response = requests.get(url, timeout=30)
            if response.status_code == 200:
                return response
            if 400 <= response.status_code < 500:
                return response  # Don't retry client errors
            if response.status_code >= 500 and attempt < max_retries:
                wait = backoff * (2 ** attempt)
                time.sleep(wait)
        except requests.RequestException:
            if attempt < max_retries:
                wait = backoff * (2 ** attempt)
                time.sleep(wait)
    return None

def fetch_all_pages(base_url, headers=None):
    """Fetch all paginated results"""
    all_data = []
    page = 1
    while True:
        params = {'page': page, 'limit': 100}
        response = requests.get(base_url, headers=headers, params=params)
        if response.status_code != 200:
            break
        data = response.json()
        if not data or len(data) == 0:
            break
        all_data.extend(data)
        page += 1
        time.sleep(0.1)  # Rate limiting
    return all_data

# Convert JSON to DataFrame
sample_data = [
    {'id': 1, 'name': 'Alice', 'age': 30, 'department': 'Engineering'},
    {'id': 2, 'name': 'Bob', 'age': 25, 'department': 'Sales'}
]
df = pd.DataFrame(sample_data)

# Handle nested JSON
nested_data = [
    {'id': 1, 'name': 'Product A', 'metadata': {'price': 99.99, 'category': 'Electronics'}},
    {'id': 2, 'name': 'Product B', 'metadata': {'price': 149.99, 'category': 'Electronics'}}
]
df_nested = pd.json_normalize(nested_data, sep='_')
print(df_nested)