https client fault in ESP-IDF (but working well with Arduino IDE)

[fjburgoa]

Hi

I'm trying to GET an answer from a server at the following address:

https://apidatos.ree.es/es/datos/mercad ... peninsular

With a browser it works fine and the answer is received. The same is true with IDE Arduino,
but I'm not able to get an answer with ESP-IDF SDK.

With Arduino:

Code: Select all

void get_pvpc(void)
{
    HTTPClient http;
    String serverName = "https://apidatos.ree.es/es/datos/mercados/precios-mercados-tiempo-real?start_date=2024-11-16T00:00&end_date=2024-11-16T23:59&time_trunc=hour&geo_limit=peninsular";

    http.begin(serverName.c_str());    // Your Domain name with URL path or IP address with path   
    int httpResponseCode = http.GET(); // Send HTTP GET request
      
    if (httpResponseCode>0) 
    {
        String payload = http.getString();
    }
    else 
    {
        Serial.println(httpResponseCode);
    }    
    http.end();  // Free resources
}

With ESP-IDF: based on sample esp_http_client_example.c

Code: Select all

/* ESP HTTP Client Example

   This example code is in the Public Domain (or CC0 licensed, at your option.)

   Unless required by applicable law or agreed to in writing, this
   software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
   CONDITIONS OF ANY KIND, either express or implied.
*/

#include <string.h>
#include <sys/param.h>
#include <stdlib.h>
#include <ctype.h>
#include "esp_log.h"
#include "nvs_flash.h"
#include "esp_event.h"
#include "esp_netif.h"
#include "protocol_examples_common.h"
#include "protocol_examples_utils.h"
#include "esp_tls.h"
#include "esp_crt_bundle.h"


#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_system.h"

#include "esp_http_client.h"

#define MAX_HTTP_RECV_BUFFER 4096
#define MAX_HTTP_OUTPUT_BUFFER 4096
static const char *TAG = ">>";

/* Root cert for howsmyssl.com, taken from howsmyssl_com_root_cert.pem

   The PEM file was extracted from the output of this command:
   openssl s_client -showcerts -connect www.howsmyssl.com:443 </dev/null

   The CA root cert is the last cert given in the chain of certs.

   To embed it in the app binary, the PEM file is named
   in the component.mk COMPONENT_EMBED_TXTFILES variable.
*/
extern const char howsmyssl_com_root_cert_pem_start[] asm("_binary_howsmyssl_com_root_cert_pem_start");
extern const char howsmyssl_com_root_cert_pem_end[]   asm("_binary_howsmyssl_com_root_cert_pem_end");

extern const char postman_root_cert_pem_start[] asm("_binary_postman_root_cert_pem_start");
extern const char postman_root_cert_pem_end[]   asm("_binary_postman_root_cert_pem_end");

esp_err_t _http_event_handler(esp_http_client_event_t *evt)
{
    static char *output_buffer;  // Buffer to store response of http request from event handler
    static int output_len;       // Stores number of bytes read
    switch(evt->event_id) {
        case HTTP_EVENT_ERROR:
            ESP_LOGD(TAG, "HTTP_EVENT_ERROR");
            break;
        case HTTP_EVENT_ON_CONNECTED:
            ESP_LOGD(TAG, "HTTP_EVENT_ON_CONNECTED");
            break;
        case HTTP_EVENT_HEADER_SENT:
            ESP_LOGD(TAG, "HTTP_EVENT_HEADER_SENT");
            break;
        case HTTP_EVENT_ON_HEADER:
            ESP_LOGD(TAG, "HTTP_EVENT_ON_HEADER, key=%s, value=%s", evt->header_key, evt->header_value);
            break;
        case HTTP_EVENT_ON_DATA:
            ESP_LOGD(TAG, "HTTP_EVENT_ON_DATA, len=%d", evt->data_len);

            if (output_len == 0 && evt->user_data) {
                // we are just starting to copy the output data into the use
                memset(evt->user_data, 0, MAX_HTTP_OUTPUT_BUFFER);
            }
            if (!esp_http_client_is_chunked_response(evt->client)) {
                // If user_data buffer is configured, copy the response into the buffer
                int copy_len = 0;
                if (evt->user_data) {
                    // The last byte in evt->user_data is kept for the NULL character in case of out-of-bound access.
                    copy_len = MIN(evt->data_len, (MAX_HTTP_OUTPUT_BUFFER - output_len));
                    if (copy_len) {
                        memcpy(evt->user_data + output_len, evt->data, copy_len);
                    }
                } else {
                    int content_len = esp_http_client_get_content_length(evt->client);
                    if (output_buffer == NULL) {
                        // We initialize output_buffer with 0 because it is used by strlen() and similar functions therefore should be null terminated.
                        output_buffer = (char *) calloc(content_len + 1, sizeof(char));
                        output_len = 0;
                        if (output_buffer == NULL) {
                            ESP_LOGE(TAG, "Failed to allocate memory for output buffer");
                            return ESP_FAIL;
                        }
                    }
                    copy_len = MIN(evt->data_len, (content_len - output_len));
                    if (copy_len) {
                        memcpy(output_buffer + output_len, evt->data, copy_len);
                    }
                }
                output_len += copy_len;
            }
            break;
        case HTTP_EVENT_ON_FINISH:
            ESP_LOGD(TAG, "HTTP_EVENT_ON_FINISH");
            if (output_buffer != NULL) {
                // Response is accumulated in output_buffer. Uncomment the below line to print the accumulated response
                // ESP_LOG_BUFFER_HEX(TAG, output_buffer, output_len);
                free(output_buffer);
                output_buffer = NULL;
            }
            output_len = 0;
            break;
        case HTTP_EVENT_DISCONNECTED:
            ESP_LOGI(TAG, "HTTP_EVENT_DISCONNECTED");
            int mbedtls_err = 0;
            esp_err_t err = esp_tls_get_and_clear_last_error((esp_tls_error_handle_t)evt->data, &mbedtls_err, NULL);
            if (err != 0) {
                ESP_LOGI(TAG, "Last esp error code: 0x%x", err);
                ESP_LOGI(TAG, "Last mbedtls failure: 0x%x", mbedtls_err);
            }
            if (output_buffer != NULL) {
                free(output_buffer);
                output_buffer = NULL;
            }
            output_len = 0;
            break;
        case HTTP_EVENT_REDIRECT:
            ESP_LOGD(TAG, "HTTP_EVENT_REDIRECT");
            esp_http_client_set_header(evt->client, "From", "user@example.com");
            esp_http_client_set_header(evt->client, "Accept", "text/html");
            esp_http_client_set_redirection(evt->client);
            break;

    }
    return ESP_OK;
}

static void http_rest_with_url(void)
{
    // Declare local_response_buffer with size (MAX_HTTP_OUTPUT_BUFFER + 1) to prevent out of bound access when
    // it is used by functions like strlen(). The buffer should only be used upto size MAX_HTTP_OUTPUT_BUFFER
    char local_response_buffer[MAX_HTTP_OUTPUT_BUFFER + 1] = {0};
    /**
     * NOTE: All the configuration parameters for http_client must be spefied either in URL or as host and path parameters.
     * If host and path parameters are not set, query parameter will be ignored. In such cases,
     * query parameter should be specified in URL.
     *
     * If URL as well as host and path parameters are specified, values of host and path will be considered.
     */
    esp_http_client_config_t config = {
        .host = "https://apidatos.ree.es/es/datos/mercados/precios-mercados-tiempo-real?start_date=2024-11-16T00:00&end_date=2024-11-16T23:59&time_trunc=hour&geo_limit=peninsular",        
        //.host = CONFIG_EXAMPLE_HTTP_ENDPOINT,
        .path = "/get",
        .query = "esp",
        .event_handler = _http_event_handler,
        .user_data = local_response_buffer,        // Pass address of local buffer to get response
        .disable_auto_redirect = true,
    };
    esp_http_client_handle_t client = esp_http_client_init(&config);

    // GET
    esp_err_t err = esp_http_client_perform(client);
    if (err == ESP_OK) {
        ESP_LOGI(TAG, "HTTP GET Status = %d, content_length = %"PRId64,
                esp_http_client_get_status_code(client),
                esp_http_client_get_content_length(client));
    } else {
        ESP_LOGE(TAG, "HTTP GET request failed: %s", esp_err_to_name(err));
    }
    ESP_LOG_BUFFER_HEX(TAG, local_response_buffer, strlen(local_response_buffer));
 
    esp_http_client_cleanup(client);
}
 

static void http_test_task(void *pvParameters)
{
    http_rest_with_url();
    ESP_LOGI(TAG, "Finish http example");

    vTaskDelete(NULL);
}


void app_main(void)
{
    esp_err_t ret = nvs_flash_init();
    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
      ESP_ERROR_CHECK(nvs_flash_erase());
      ret = nvs_flash_init();
    }
    ESP_ERROR_CHECK(ret);

    ESP_ERROR_CHECK(esp_netif_init());
    ESP_ERROR_CHECK(esp_event_loop_create_default());
 
    ESP_ERROR_CHECK(example_connect());
    ESP_LOGI(TAG, "Connected to AP, begin http example");


    xTaskCreate(&http_test_task, "http_test_task", 2*8192, NULL, 5, NULL);

    
    while (1)
    {
        vTaskDelay(1000/portTICK_PERIOD_MS); 
    }
 
}

The answer I get is:

I (3522) wifi:dp: 1, bi: 102400, li: 3, scale listen interval from 307200 us to 307200 us
I (3532) wifi:set rx beacon pti, rx_bcn_pti: 0, bcn_timeout: 25000, mt_pti: 0, mt_time: 10000
I (3582) wifi:AP's beacon interval = 102400 us, DTIM period = 1
I (4552) wifi:<ba-add>idx:0 (ifx:0, f4:23:9c:0d:d9:81), tid:0, ssn:0, winSize:64
I (5442) example_connect: Got IPv6 event: Interface "example_netif_sta" address: fe80:0000:0000:0000:66e8:33ff:fe54:41e0, type: ESP_IP6_ADDR_IS_LINK_LOCAL
I (6542) esp_netif_handlers: example_netif_sta ip: 192.168.0.22, mask: 255.255.255.0, gw: 192.168.0.1
I (6542) example_connect: Got IPv4 event: Interface "example_netif_sta" address: 192.168.0.22
I (6552) example_common: Connected to example_netif_sta
I (6552) example_common: - IPv4 address: 192.168.0.22,
I (6562) example_common: - IPv6 address: fe80:0000:0000:0000:66e8:33ff:fe54:41e0, type: ESP_IP6_ADDR_IS_LINK_LOCAL
I (6572) >>: Connected to AP, begin http example
E (13572) esp-tls: couldn't get hostname for :https://apidatos.ree.es/es/datos/mercad ... peninsular: getaddrinfo() returns 202, addrinfo=0x0
E (13582) transport_base: Failed to open a new connection: 32769
E (13582) HTTP_CLIENT: Connection failed, sock < 0
E (13592) >>: HTTP GET request failed: ESP_ERR_HTTP_CONNECT
I (13602) >>: HTTP_EVENT_DISCONNECTED
I (13602) >>: Last esp error code: 0x8001
I (13602) >>: Last mbedtls failure: 0x0
I (13612) >>: Finish http example


Any help is wellcome!!!

Many Thanks

[nopnop2002]

If you use https, you need to set cert_pem.

https://github.com/espressif/esp-idf/bl ... ple.c#L554

Root cert can be created by following the steps below.

https://github.com/espressif/esp-idf/bl ... mple.c#L35

Code: Select all

$ openssl s_client -showcerts -connect apidatos.ree.es:443 < /dev/null

[MicroController]

Code: Select all

.host = "https://apidatos.ree.es/es/datos/..."

This is not a host/server name...

[fjburgoa]

Hi, Many thanks to both of you for the suggestions and the support

I got the certificate according the instructions.
I overwritted the existing howmyssl_com_root_cert.pem and postman_root_cert.pem with the cert from the site:

-----BEGIN CERTIFICATE-----
MIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G
A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp
....

-----END CERTIFICATE-----


But I'm still having some difficulties to get the answer.

I receive the following message:



I (3539) wifi:dp: 1, bi: 102400, li: 3, scale listen interval from 307200 us to 307200 us
I (3539) wifi:set rx beacon pti, rx_bcn_pti: 0, bcn_timeout: 25000, mt_pti: 0, mt_time: 10000
I (3549) wifi:AP's beacon interval = 102400 us, DTIM period = 1
I (4649) wifi:<ba-add>idx:0 (ifx:0, f4:23:9c:0d:d9:81), tid:0, ssn:0, winSize:64
I (5449) example_connect: Got IPv6 event: Interface "example_netif_sta" address: fe80:0000:0000:0000:66e8:33ff:fe54:41e0, type: ESP_IP6_ADDR_IS_LINK_LOCAL
I (6559) esp_netif_handlers: example_netif_sta ip: 192.168.0.22, mask: 255.255.255.0, gw: 192.168.0.1
I (6559) example_connect: Got IPv4 event: Interface "example_netif_sta" address: 192.168.0.22
I (6569) example_common: Connected to example_netif_sta
I (6569) example_common: - IPv4 address: 192.168.0.22,
I (6579) example_common: - IPv6 address: fe80:0000:0000:0000:66e8:33ff:fe54:41e0, type: ESP_IP6_ADDR_IS_LINK_LOCAL
I (6589) >>: Connected to AP, begin http example
I (6599) main_task: Returned from app_main()
I (6699) esp-x509-crt-bundle: Certificate validated
I (7469) >>: HTTPS Status = 200, content_length = -1
I (7469) >>: HTTP_EVENT_DISCONNECTED
I (8389) >>: HTTPS Status = 200, content_length = -1
I (8389) >>: HTTP_EVENT_DISCONNECTED



The code looks like now as follows:

Code: Select all

#define MAX_HTTP_RECV_BUFFER 2048
#define MAX_HTTP_OUTPUT_BUFFER 2048
static const char *TAG = ">>";

extern const char ree_root_cert_pem_start[] asm("_binary_ree_root_cert_pem_start");
extern const char ree_root_cert_pem_end[]   asm("_binary_ree_root_cert_pem_end");

//--------------------------------------------------------------
esp_err_t _http_event_handler(esp_http_client_event_t *evt)
{
  *** No change here with regards to the example ***
}

//--------------------------------------------------------------
static void https_with_url(void)
{
    char local_response_buffer[MAX_HTTP_OUTPUT_BUFFER + 1] = {0};

    esp_http_client_config_t config = {
        .url = "https://apidatos.ree.es/es/datos/mercados/precios-mercados-tiempo-real?start_date=2024-11-16T00:00&end_date=2024-11-16T23:59&time_trunc=hour&geo_limit=peninsular",        
        .transport_type = HTTP_TRANSPORT_OVER_SSL,
        .method = HTTP_METHOD_GET,
        .event_handler = _http_event_handler,
        [b].crt_bundle_attach = esp_crt_bundle_attach,[/b]
        .user_data = local_response_buffer,        // Pass address of local buffer to get response
        //.disable_auto_redirect = true,
    };
    esp_http_client_handle_t client = esp_http_client_init(&config);
    esp_err_t err = esp_http_client_perform(client);

    if (err == ESP_OK) {
        ESP_LOGI(TAG, "HTTPS Status = %d, content_length = %"PRId64,
                esp_http_client_get_status_code(client),
                esp_http_client_get_content_length(client));
    } else {
        ESP_LOGE(TAG, "Error perform http request %s", esp_err_to_name(err));
    }
    esp_http_client_cleanup(client);
}
 
//--------------------------------------------------------------
static void https_with_hostname_path(void)
{
    char local_response_buffer[MAX_HTTP_OUTPUT_BUFFER + 1] = {0};

    esp_http_client_config_t config = {
        .url = "https://apidatos.ree.es/es/datos/mercados/precios-mercados-tiempo-real?start_date=2024-11-16T00:00&end_date=2024-11-16T23:59&time_trunc=hour&geo_limit=peninsular",
        .method = HTTP_METHOD_GET,
        .transport_type = HTTP_TRANSPORT_OVER_SSL,
        .event_handler = _http_event_handler,
        //.cert_pem = howsmyssl_com_root_cert_pem_start,
        //.cert_pem = postman_root_cert_pem_start,
        [b].cert_pem = ree_root_cert_pem_start,[/b]
        .user_data = local_response_buffer,        // Pass address of local buffer to get response
    };
    esp_http_client_handle_t client = esp_http_client_init(&config);
    esp_err_t err = esp_http_client_perform(client);

    if (err == ESP_OK) {
        ESP_LOGI(TAG, "HTTPS Status = %d, content_length = %"PRId64,
                esp_http_client_get_status_code(client),
                esp_http_client_get_content_length(client));
    } else {
        ESP_LOGE(TAG, "Error perform http request %s", esp_err_to_name(err));
    }
    esp_http_client_cleanup(client);
}

//--------------------------------------------------------------
static void http_test_task(void *pvParameters)
{
    https_with_url();                           //one
    https_with_hostname_path();      //two
    vTaskDelete(NULL);
}
//--------------------------------------------------------------
void app_main(void)
{
    esp_err_t ret = nvs_flash_init();
    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) 
    {
      ESP_ERROR_CHECK(nvs_flash_erase());
      ret = nvs_flash_init();
    }
    ESP_ERROR_CHECK(ret);

    ESP_ERROR_CHECK(esp_netif_init());
    ESP_ERROR_CHECK(esp_event_loop_create_default());
    ESP_ERROR_CHECK(example_connect());
    ESP_LOGI(TAG, "Connected to AP, begin http example");

    xTaskCreate(&http_test_task, "http_test_task", 8192, NULL, 5, NULL);
}

By the way, using .host it doesn't work neither, it gets an exception and reboots.


Again, many many thanks for your help

Javier

[MicroController]

Code: Select all

esp_http_client_config_t config = { ...

You should either explicitly initialize all fields of the structure, or clear the structure to all-0 before setting the fields you want.

[Sprite]

Code: Select all

esp_http_client_config_t config = { ...

You should either explicitly initialize all fields of the structure, or clear the structure to all-0 before setting the fields you want.

No, the structure he used actually is fine and the compiler will zero-initialize unnamed fields. The issue is when you do something like

Code: Select all

esp_http_client_config_t config;
config.host="bla";
config.port=80;

Then you will get uninitialized fields.

[fjburgoa]

Hi, I tried with the initialization by fields, but again the same feedback:

Code: Select all

static void https_with_url(void)
{
    char local_response_buffer[MAX_HTTP_OUTPUT_BUFFER + 1] = {0};

    esp_http_client_config_t config;
    memset(&config, 0, sizeof(esp_http_client_config_t));
    config.url = "https://apidatos.ree.es/es/datos/mercados/precios-mercados-tiempo-real?start_date=2024-11-16T00:00&end_date=2024-11-16T23:59&time_trunc=hour&geo_limit=peninsular";
    config.transport_type = HTTP_TRANSPORT_OVER_SSL;
    config.method = HTTP_METHOD_GET;
    config.event_handler = _http_event_handler;
    config.timeout_ms = 5000;
    config.port = 443;
    config.crt_bundle_attach = esp_crt_bundle_attach;
    config.user_data = local_response_buffer;        // Pass address of local buffer to get response
    config.disable_auto_redirect = true;

    esp_http_client_handle_t client = esp_http_client_init(&config);
    esp_err_t err = esp_http_client_perform(client);

    if (err == ESP_OK) {
        ESP_LOGI(TAG, "HTTPS Status = %d, content_length = %"PRId64, esp_http_client_get_status_code(client), esp_http_client_get_content_length(client));
        printf("local_response_buffer: %s\n", local_response_buffer);
    } else {
        ESP_LOGE(TAG, "Error perform http request %s", esp_err_to_name(err));
    }
    esp_http_client_cleanup(client);
}

Output

I (6559) esp_netif_handlers: example_netif_sta ip: 192.168.0.22, mask: 255.255.255.0, gw: 192.168.0.1
I (6559) example_connect: Got IPv4 event: Interface "example_netif_sta" address: 192.168.0.22
I (6559) example_common: Connected to example_netif_sta
I (6569) example_common: - IPv4 address: 192.168.0.22,
I (6579) example_common: - IPv6 address: fe80:0000:0000:0000:66e8:33ff:fe54:41e0, type: ESP_IP6_ADDR_IS_LINK_LOCAL
I (6589) >>: Connected to AP, begin http example
I (6589) main_task: Returned from app_main()
I (6669) esp-x509-crt-bundle: Certificate validated
I (7519) >>: HTTPS Status = 200, content_length = -1
local_response_buffer:
I (7519) >>: HTTP_EVENT_DISCONNECTED


I've to say it is really weird as. The code in Arduino IDE doesn't use any secure request, it is just a http GET.
But here with ESP-IDF looks much more complex.

In a browser (firefox, chrome). If you enter http://apidatos.ree.es/es/datos/mercado ... peninsular", the request is redirected to an https service, but it still works.


Many thanks.

Javier