Databricks Fundamentals

# Installeer eerst: pip install databricks-connect
from databricks import sql
import pandas as pd

def connect_to_databricks(server_hostname, http_path, access_token):
    """
    Maak verbinding met Databricks SQL Endpoint
    
    Parameters:
    - server_hostname: je-databricks-workspace.cloud.databricks.com
    - http_path: /sql/1.0/warehouses/your-warehouse-id
    - access_token: persoonlijke access token
    """
    connection = sql.connect(
        server_hostname=server_hostname,
        http_path=http_path,
        access_token=access_token
    )
    
    print("✅ Succesvol verbonden met Databricks!")
    return connection

def run_sql_query(connection, query):
    """
    Voer SQL query uit op Databricks
    """
    cursor = connection.cursor()
    cursor.execute(query)
    
    # Fetch results
    result = cursor.fetchall()
    columns = [desc[0] for desc in cursor.description]
    
    # Convert to pandas DataFrame
    df = pd.DataFrame(result, columns=columns)
    
    cursor.close()
    return df

# Voorbeeld gebruik
if __name__ == "__main__":
    # Vervang met je eigen credentials
    conn = connect_to_databricks(
        server_hostname="your-workspace.cloud.databricks.com",
        http_path="/sql/1.0/warehouses/abc123",
        access_token="dapi1234567890abcdef"
    )
    
    # Voorbeeld query
    query = """
    SELECT 
        category,
        COUNT(*) as product_count,
        AVG(price) as avg_price,
        SUM(sales) as total_sales
    FROM catalog.schema.products
    WHERE year = 2024
    GROUP BY category
    ORDER BY total_sales DESC
    LIMIT 10
    """
    
    results_df = run_sql_query(conn, query)
    print(results_df.head())
    conn.close()