Core Panic with Bluetooth + freeRTOS

[paeion]

Code: Untitled.txt Select all

Hi all, I have a simple program to measure an external ADC via SPI with the ESP32 and I am trying to send the raw data received over bluetooth to my PC. I have implemented freeRTOS within the arduino IDE and when using the Serial port the code works without issues. When I attempt to do the same using Bluetooth Serial the ESP32 resets continuously in panic and I get: Guru Meditation Error: Core  1 panic'ed (Interrupt wdt timeout on CPU1). I have tried my best to debug it and explore similar threads on the internet but I can't really figure out the issue. Below is my code and the snippet of the error message (I haven't figured out how to decode the backtrace yet). Any advice would be extremely welcome, thanks in advance! 

Code: Untitled.cpp Select all

// 

#include "esp_system.h" 

#include "freertos/FreeRTOS.h"

#include "freertos/task.h"

#include "freertos/timers.h"



#include <SPI.h>

#include <SoftwareSerial.h>

#include <BluetoothSerial.h>

#include <ads1262.h>



#define PGA 32          // Programmable Gain = 32

#define VREF 2.50       // Internal reference of 2.5V

#define BUFF_SIZE 32

#define BYTE_LEN 4      // Number of bytes for 32 bit int



ads1262 ADS; // ADS class

BluetoothSerial SerialBT;



int32_t val1;

int32_t val2;

int32_t val3;

int32_t val4;

int32_t val5;



int32_t data1;

int32_t data2;

int32_t data3;

int32_t data4;

int32_t timeData;



int32_t t_start;

int32_t t_finish;



volatile char *data_struct;

volatile char buff_array[6];

volatile long data_array[4];

volatile signed long counts;

volatile int isDataReady = 0;



TaskHandle_t TaskGetData;

TaskHandle_t TasksendPC;

QueueHandle_t Queue1;

QueueHandle_t Queue2;

QueueHandle_t Queue3;

QueueHandle_t Queue4;

QueueHandle_t QueueT;

//SemaphoreHandle_t semaphore; 



void setup()

{ 

    portDISABLE_INTERRUPTS();



    // initalize the  data ready and chip select pins:

    pinMode(ADS1262_DRDY_PIN, INPUT);   // Data ready input line

    pinMode(ADS1262_CS_PIN, OUTPUT);    // Chip enable output line

    pinMode(ADS1262_START_PIN, OUTPUT); // Start

    pinMode(ADS1262_PWDN_PIN, OUTPUT);  // Power down output

    attachInterrupt(digitalPinToInterrupt(ADS1262_DRDY_PIN), awaitDRDY, FALLING);



    SerialBT.begin("PLAKA PLATES");

//    Serial.begin(500000);               // Max Baud Rate

    ADS.ads1262_Init();                 // Initialise ADS1262



    Queue1 = xQueueCreate(500, sizeof(int32_t));

    Queue2 = xQueueCreate(500, sizeof(int32_t));

    Queue3 = xQueueCreate(500, sizeof(int32_t));

    Queue4 = xQueueCreate(500, sizeof(int32_t));

    QueueT = xQueueCreate(500, sizeof(int32_t));



   setCpuFrequencyMhz(240);

   xTaskCreatePinnedToCore(loopGet, "TaskGetData", 4096, NULL, 3, &TaskGetData, 1); // 0

   xTaskCreatePinnedToCore(loopSend, "TasksendPC", 4096, NULL, 3, &TasksendPC, 1); // 1

   

   portENABLE_INTERRUPTS();

}



void loop()

{

    vTaskDelete (NULL);

}



 void loopGet(void *parameter)

{ 

    for (;;) {

        t_start = micros(); // micros()



        // Cycle through differential channels

        val1 = read_channel(0x01);

        val2 = read_channel(0x23);

        val3 = read_channel(0x45);

        val4 = read_channel(0x67);

        val5 = read_channel(0x01);  // Dummy read to not affect 4th load cell reading

        

        xQueueSend(Queue1, &val1, portMAX_DELAY);

        xQueueSend(Queue2, &val2, portMAX_DELAY);

        xQueueSend(Queue3, &val3, portMAX_DELAY);

        xQueueSend(Queue4, &val4, portMAX_DELAY);

        

        t_finish = micros() - t_start;

        xQueueSend(QueueT, &t_finish, portMAX_DELAY);

    }

}



 void loopSend(void *parameter)

{ 

    for (;;) {

  

        xQueueReceive(Queue1, &data1, portMAX_DELAY);

        xQueueReceive(Queue2, &data2, portMAX_DELAY);

        xQueueReceive(Queue3, &data3, portMAX_DELAY);

        xQueueReceive(Queue4, &data4, portMAX_DELAY);

        xQueueReceive(QueueT, &timeData, portMAX_DELAY);



        // Serial Port Debug

//        printf("%i,%i,%i,%i,%i\n", data1, data2, data3, data4, timeData);

        

        // Send binary to Bluetooth Serial Port

        SerialBT.write((byte*)&data1, BYTE_LEN);

        SerialBT.write((byte*)&data2, BYTE_LEN);

        SerialBT.write((byte*)&data3, BYTE_LEN);

        SerialBT.write((byte*)&data4, BYTE_LEN);

        SerialBT.write((byte*)&timeData, BYTE_LEN);

        SerialBT.write('\n'); 

    }

}



void awaitDRDY() {  // IRAM_ATTR

    isDataReady = 1;

//    xSemaphoreGive(semaphore);

}



// Read adc binary data

int32_t read_channel(char channel)

{

    // Channel multiplexer channel

    ADS.ads1262_Reg_Write(INPMUX, channel);

   

    // Wait until ADC is ready for conversion    //    while (digitalRead(ADS1262_DRDY_PIN) != LOW){

    isDataReady = 0;

    while (isDataReady != 1) {}

//    xSemaphoreTake(semaphore, portMAX_DELAY);  // Faster ISR response to DRDY signal 

    

    // Dump Binary Data

    data_struct = ADS.ads1262_Read_Data();

    for (int i = 1; i < 5; ++i)

      {

        buff_array[i] = *(data_struct + i);              

        data_array[i] = (unsigned char)buff_array[i];

      }

    counts = (signed long) (((unsigned long)data_array[0]<<24) | ((unsigned long)data_array[1]<<16) | (data_array[2]<<8) | data_array[3]);

    return counts;

}

Code: Untitled.txt Select all

Rebooting...

ets Jul 29 2019 12:21:46



rst:0xc (SW_CPU_RESET),boot:0x13 (SPI_FAST_FLASH_BOOT)

configsip: 0, SPIWP:0xee

clk_drv:0x00,q_drv:0x00,d_drv:0x00,cs0_drv:0x00,hd_drv:0x00,wp_drv:0x00

mode:DIO, clock div:1

load:0x3fff0018,len:4

load:0x3fff001c,len:1044

load:0x40078000,len:10124

load:0x40080400,len:5856

entry 0x400806a8

Guru Meditation Error: Core  1 panic'ed (Interrupt wdt timeout on CPU1)

Core 1 register dump:

PC      : 0x40092824  PS      : 0x00060b34  A0      : 0x80091a3b  A1      : 0x3ffc7490  

A2      : 0x3ffbbca4  A3      : 0x3ffc7720  A4      : 0x00000001  A5      : 0x00000001  

A6      : 0x00060b23  A7      : 0x00000000  A8      : 0x3ffc7720  A9      : 0x3ffc7720  

A10     : 0x00000018  A11     : 0x00000018  A12     : 0x00000001  A13     : 0x00000001  

A14     : 0x00060b23  A15     : 0x00000000  SAR     : 0x0000001d  EXCCAUSE: 0x00000006  

EXCVADDR: 0x00000000  LBEG    : 0x4000c2e0  LEND    : 0x4000c2f6  LCOUNT  : 0xffffffff  



ELF file SHA256: 0000000000000000



Backtrace: 0x40092824:0x3ffc7490 0x40091a38:0x3ffc74b0 0x4008fcef:0x3ffc74d0 0x400e2fbb:0x3ffc7510 0x4015a1df:0x3ffc7530 0x400e354f:0x3ffc7570 0x400d23aa:0x3ffc7590 0x400d1e3e:0x3ffc75d0 0x400d13ba:0x3ffc7610 0x400d3332:0x3ffc7650 0x4008ff1a:0x3ffc7670



Core 0 register dump:

PC      : 0x40090f1e  PS      : 0x00060334  A0      : 0x80092079  A1      : 0x3ffcbb10  

A2      : 0x3ffbed5c  A3      : 0x0000cdcd  A4      : 0xb33fffff  A5      : 0x00000001  

A6      : 0x00060123  A7      : 0x0000abab  A8      : 0x0000abab  A9      : 0x3ffcbc50  

A10     : 0x3ffae704  A11     : 0x00000002  A12     : 0x00060320  A13     : 0x3ffcbcf8  

A14     : 0x00000006  A15     : 0x00000000  SAR     : 0x00000000  EXCCAUSE: 0x00000006  

EXCVADDR: 0x00000000  LBEG    : 0x4000c2e0  LEND    : 0x4000c2f6  LCOUNT  : 0x00000000  



ELF file SHA256: 0000000000000000



Backtrace: 0x40090f1e:0x3ffcbb10 0x40092076:0x3ffcbb40 0x4008fc57:0x3ffcbb60 0x40086517:0x3ffcbba0 0x400865ed:0x3ffcbbd0 0x40172691:0x3ffcbbf0 0x40155636:0x3ffcbc10 0x401556e5:0x3ffcbc50 0x400814e6:0x3ffcbc80 0x40159676:0x3ffcbca0 0x40159744:0x3ffcbcd0 0x40159d28:0x3ffcbcf0 0x4008ff1a:0x3ffcbd20

[Sprite]

Code: Untitled.cpp Select all

// 

void setup()

{ 

    portDISABLE_INTERRUPTS();

[...]   

   portENABLE_INTERRUPTS();

}

Why?

[paeion]

ESP_Sprite, I was attempting to use a binary semaphore to remove the while loop that waits for the external interrupt and I believe

Code: Untitled.cpp Select all

xSemaphoreGive(semaphore);

was calling before the xSemaphoreCreate() line was executed in setup hence disabling the interrupts. I believe it is not needed for how I am doing it and just something I forgot to comment out.

[Sprite]

Good. Does removing those lines also fix your issue? If not, decoding the backtrace is your best bet: you probably want this.

[paeion]

Thanks ESP_Sprite, admittedly removing those two lines fixed the core panic issue I was seeing. I had to move around the sequence of initialisation in the setup() to get the Bluetooth to work properly. For example, sometimes I cannot connect to the ESP32 bluetooth port on my Mac saying 'resource busy'.

I am also experiencing the following behaviour which is very confusing:
Case 1: I send the ADC data over the software serial port, the data is good.
Case 2: I send the data over the software AND bluetooth serial, the data is good ONLY IF viewed over the software serial port.
Case 3: I do as above and open the software AND bluetooth port, the data is corrupted at random, recurrent intervals. As soon as the bluetooth port is closed the data is good again on the software serial.
Case 4: I only send data over the bluetooth serial port, the same periodic data corruption is present.
(Attached photos of behaviour)

I know for a fact that if the ADC data read routine does not happen fast enough, the data can be corrupted because the SPI communication is reading the ADC register as it settles to new values. However I cannot understand how opening the bluetooth port could case additional latency in the ESP freeRTOS if the bluetooth serial data is being sent regardless of the activity on the PC side (i.e. port open or closed).

Any thoughts on this would be a lifesaver!
(For reference: Using Arduino IDE on Windows 10 to flash ESP32 Firebeetle board, software serial port open on Arduino IDE/ PuTTy on Windows 10, bluetooth serial open on MacOS High Sierra using Python 3 and the PySerial library)

[Sprite]

I can't help you with that, sorry, I'm not deeply familiar with those Arduino APIs.

[paeion]

All good ESP_Sprite! I managed to debug it, it has nothing to do with the functioning of the ESP32 or the bluetooth. I was using "\n" newline characters that are showing up in my data and therefore getting split data packets when I should not have. Thanks again for the rapid responses and assistance!

[xien551]

Hey, paeion. Arduino is not a real time system. I remember the timestamp is only 1ms. I doubt that the Arduino would not work well with freeRTOS. And I suggest using IDF instead. I think you should learn the callback function, which is also popular in STM32 SDK for better real-time performance.

[Sprite]

Hey, paeion. Arduino is not a real time system. I remember the timestamp is only 1ms. I doubt that the Arduino would not work well with freeRTOS. And I suggest using IDF instead. I think you should learn the callback function, which is also popular in STM32 SDK for better real-time performance.

Can you substantiate that 'Arduino is not a real-time system'? It's a strange assertion to me, especially given that it's built on ESP-IDF itself.

[xien551]

Let me tell you the history of Arduino.

Do you know that Arduino is developed by the artist?
Some artists want to design a Automatic works of art. Namely, the work is not static, but can move, like a robot.

They find many microprocessor, just find them hard to learn. So they create the Arduino, which is not just a software, it's a system with hardware and software. The original hardware is Arduino uno, which is based on ATmega 32, a 8 bit microprocessor. The software is what we familiar with ,the Arduino ide, which is a editor, compiler, flash tool and monitor.

Arduino is not used for real time application. the artist work need not to be acted very fast, 1ms is enough.
Arduino hide hardware interrupts underlying code, which is important to real time performance.

Also, real-time performance is different to different application. For the artist, 1ms is realy precise. But, for industrial application, 1ms is too slow, we want the 1us precision. They need real time operation system. For example, VxWorks, rtLinux，ucos, freeRTOS. If 1us is not enough, then we choose the hardware solution, CPLD or FPGA.

Arduino is not suitable for industrial application, where the us realtime performance is important. For most critical real-time application, even the mcu or dsp is not used, we use PLC.

real-time is not easy to realize, freeRTOS is a good system. But Arduino is not ,they are just used for prototype verification. If you are a engineer, you would agree with me. No one dare using Arduino for industrial application, for Example, Power system.