Trying to make a REST Call to get steps details - API Version not supported

Hello,

I am trying to use Azure OpenAI from a Databricks notebook to generate PySpark code for a Bronze Loader Agent.

i am getting azure open ai endpoint not supporting Bad Request Error API version not supported error message .
for sample this is mycode

My code:

client = AzureOpenAI(
    api_key="xxxx",
    api_version="2024-11-20",
    azure_endpoint="https://xxx.services.ai.azure.com/api/projects/proj-default"
)

def call_llm(prompt):
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": prompt}]
    )
    
    content = response.choices[0].message.content.strip()

    # HARD CLEANING (VERY IMPORTANT)
    if content.lower().startswith("python"):
        content = content[len("python"):].strip()

    # Remove markdown blocks
    content = content.replace("```python", "").replace("```", "").strip()

    return content


# ---------- Bronze Loader Agent ----------
def bronze_loader_agent(volume_path, catalog, schema, max_retries=3):
    
    print("\nRunning Bronze Loader Agent...")
    print("Volume Path:", volume_path)

    base_prompt = f"""
    You are a senior data engineer working in Databricks.

    Generate ONLY valid PySpark code.

    Task:
    - Read all CSV files from: {volume_path}
    - Use dbutils.fs.ls()
    - Loop through files
    - Extract table name (remove .csv, lowercase)
    - Load CSV (header=True, inferSchema=True)
    - Write RAW data to Delta tables
    - delete from the volume after processing

    Target:
    {catalog}.{schema}.<table_name>

    STRICT RULES:
    - ONLY Python code
    - NO explanations
    - NO markdown
    - DO NOT use f-strings
    - Use string concatenation ONLY
    - DO NOT create SparkSession
    - Ensure code is COMPLETE and EXECUTABLE

    """

    attempt = 0
    last_error = None
    code = None

    while attempt < max_retries:
        print(f"\nAttempt {attempt + 1} generating/fixing code...\n")

        if attempt == 0:
            prompt = base_prompt
        else:
            prompt = f"""
            The following PySpark code failed during execution.

            FAILED CODE:
            {code}

            ERROR MESSAGE:
            {last_error}

            Your task:
            - Fix the code so it runs successfully
            - Keep the original intent intact
            - Ensure no syntax errors
            - Ensure variables are correct
            - Ensure complete statements

            STRICT RULES:
            - Return ONLY corrected Python code
            - NO explanations
            - NO markdown
            - DO NOT use f-strings
            - Ensure code is COMPLETE and EXECUTABLE
            """

        code = call_llm(prompt)

        # Clean markdown just in case
        code = code.replace("```python", "").replace("```", "").strip()

        print("===== GENERATED CODE =====\n", code)

        try:
            print("\nExecuting code...\n")
            exec(code, globals())

            print("\n Bronze loading completed!")
            return {
                "status": "success",
                "generated_code": code,
                "attempts": attempt + 1
            }

        except Exception as e:
            last_error = str(e)

            print(f"\n Execution failed (Attempt {attempt + 1})")
            print("Error:", last_error)

            attempt += 1

    print("\n All attempts failed!")

    return {
        "status": "failed",
        "final_code": code,
        "error": last_error,
        "attempts": max_retries
    }

cell 2:

result = bronze_loader_agent(
    volume_path="/Volumes/retail_data/bronze/raw_data_volume/",
    catalog="retail_data",
    schema="bronze"
)

print(result)

However, I receive error message:

BadRequestError: Error code: 400
{
    'error': {
        'code': 'BadRequest',
        'message': 'API version not supported'
    }
}

how to resolve this issue ? I keep getting an error message