Custom Audio Element Problem

[paddlesteamer]

Hello,

I'm trying to write my own audio element. The board I'm using is LyraT v4.3. I created a pipeline i2s_stream_reader -> my_element -> i2s_stream_writer. In the process function of 'my_element', if I forward the samples without doing anything with them, I can hear my voice from the headphones without a problem. If I multiply the samples with 0.5 before forwarding them to i2s_stream_writer, all I hear is very loud noise. Aren't the samples provided to my process function raw audio samples? Should I add some kind of decode element between i2s_stream_reader and my_element?

Here is my code for reference:

Code: Untitled.c Select all

#include <string.h>

#include "board.h"

#include "esp_log.h"

#include "audio_pipeline.h"

#include "audio_element.h"

#include "i2s_stream.h"



static const char *TAG = "ELEMENTEXAMPLE";



static esp_err_t el_open(audio_element_handle_t self) {

    ESP_LOGI(TAG, "open");

    return ESP_OK;

} 



static int el_process(audio_element_handle_t self, char *buf, int len) {

    int rsize = audio_element_input(self, buf, len);



    for (int i =0; i<rsize; i+=4) {

        // convert bytes to samples 

        uint16_t lSample = ((((uint16_t) buf[i]) << 8) & 0xFF00) + buf[i+1];

        uint16_t rSample = ((((uint16_t) buf[i+2]) << 8) & 0xFF00) + buf[i+3];



        // if i comment out these two lines it works as expected

        lSample *= 0.5;

        rSample *= 0.5;



        // convert samples to byte

        buf[i] = (char) (((lSample & 0xFF00) >> 8) & 0xFF);

        buf[i+1] = (char) (lSample & 0xFF);



        buf[i+2] = (char) (((rSample & 0xFF00) >> 8) & 0xFF);

        buf[i+3] = (char) (rSample & 0xFF);

    }



    rsize = audio_element_output(self, buf, rsize);



    return rsize;

}



void app_main() {

    audio_pipeline_handle_t pipeline;

    audio_element_handle_t i2s_stream_reader, i2s_stream_writer, my_el;



    esp_log_level_set("*", ESP_LOG_INFO);



    ESP_LOGI(TAG, "[1.0] Start codec chip");

    audio_board_handle_t board_handle = audio_board_init();

    audio_hal_ctrl_codec(board_handle->audio_hal, AUDIO_HAL_CODEC_MODE_BOTH, AUDIO_HAL_CTRL_START);

    audio_hal_set_volume(board_handle->audio_hal, 100);

   

    ESP_LOGI(TAG, "[2.0] Create i2s stream to read audio data from codec chip");

    i2s_stream_cfg_t i2sr_cfg = I2S_STREAM_CFG_DEFAULT();

    i2sr_cfg.type = AUDIO_STREAM_READER;

    i2s_stream_reader = i2s_stream_init(&i2sr_cfg);    



    ESP_LOGI(TAG, "[3.0] Create i2s stream to write audio data to codec chip");

    i2s_stream_cfg_t i2sw_cfg = I2S_STREAM_CFG_DEFAULT();

    i2sw_cfg.type = AUDIO_STREAM_WRITER;

    i2s_stream_writer = i2s_stream_init(&i2sw_cfg);



    ESP_LOGI(TAG, "[4.0] Init Element");

    audio_element_cfg_t cfg = DEFAULT_AUDIO_ELEMENT_CONFIG();

    cfg.open = el_open;

    cfg.process = el_process;

    cfg.tag = "my_el";



    my_el = audio_element_init(&cfg);



    ESP_LOGI(TAG, "[5.0] Create pipeline");

    audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG();

    pipeline = audio_pipeline_init(&pipeline_cfg);

    mem_assert(pipeline);



    audio_pipeline_register(pipeline, my_el, "my_el");

    audio_pipeline_register(pipeline, i2s_stream_reader, "i2sr");

    audio_pipeline_register(pipeline, i2s_stream_writer, "i2sw");



    const char *link_tag[3] = {"i2sr", "my_el", "i2sw"};

    audio_pipeline_link(pipeline, link_tag, 3);



    ESP_LOGI(TAG, "[6.0] Run pipeline");

    audio_pipeline_run(pipeline);



    ESP_LOGI(TAG, "[ * ] Starting ...");

    while (1) {

        vTaskDelay(2000);

    }

}

P.S: I have checked out a similar post but the example code shared in that post doesn't change the samples so it doesn't answer my question.

[paddlesteamer]

Ok, there were many things wrong with this code. Here is the correct version:

Code: Untitled.c Select all

#include <string.h>

#include "board.h"

#include "esp_log.h"

#include "audio_pipeline.h"

#include "audio_element.h"

#include "i2s_stream.h"



static const char *TAG = "ELEMENTEXAMPLE";



static esp_err_t el_open(audio_element_handle_t self) {

    ESP_LOGI(TAG, "open");

    return ESP_OK;

}

static float convertToF32(uint16_t sample) {

	float s = (float) sample;



	s -= 32768.0;

	if (s > 0.0) {

		return s / 32767.0;

	}



	return s / 32768.0;

}



static uint16_t convertToUInt16(float sample) {

	if (sample > 0.0) {

		sample *= 32767.0;

	} else {

		sample *= 32768.0;

	}



	sample += 32768.0;



	return (uint16_t) sample;

}





static int el_process(audio_element_handle_t self, char *buf, int len) {

    int rsize = audio_element_input(self, buf, len);



    if (len != rsize || (rsize % 4) != 0) {

        ESP_LOGW(TAG, "unexpected rsize: %d, len: %d", rsize, len);

    } 



    uint16_t lSample, rSample;

    char * lSamplep = (char *) &lSample;

    char * rSamplep = (char *) &rSample;

    for (int i =0; i<rsize; i+=4) {

        rSamplep[0] = buf[i];

        rSamplep[1] = buf[i+1];



        lSamplep[0] = buf[i+2];

        lSamplep[1] = buf[i+3];



        float a = convertToF32(rSample);

        a *= 0.5;

        rSample = convertToUInt16(a);



        buf[i] = rSamplep[0];

        buf[i+1] = rSamplep[1];



        buf[i+2] = lSamplep[0];

        buf[i+3] = lSamplep[1];

    }



    rsize = audio_element_output(self, buf, rsize);



    return rsize;

}



void app_main() {

    audio_pipeline_handle_t pipeline;

    audio_element_handle_t i2s_stream_reader, i2s_stream_writer, my_el;



    esp_log_level_set("*", ESP_LOG_INFO);



    ESP_LOGI(TAG, "[1.0] Start codec chip");

    audio_board_handle_t board_handle = audio_board_init();

    audio_hal_ctrl_codec(board_handle->audio_hal, AUDIO_HAL_CODEC_MODE_BOTH, AUDIO_HAL_CTRL_START);

   

    ESP_LOGI(TAG, "[2.0] Create i2s stream to read audio data from codec chip");

    i2s_stream_cfg_t i2sr_cfg = I2S_STREAM_CFG_DEFAULT();

    i2sr_cfg.i2s_config.communication_format = I2S_COMM_FORMAT_I2S_MSB;

    i2sr_cfg.type = AUDIO_STREAM_READER;

    i2s_stream_reader = i2s_stream_init(&i2sr_cfg);    



    ESP_LOGI(TAG, "[3.0] Create i2s stream to write audio data to codec chip");

    i2s_stream_cfg_t i2sw_cfg = I2S_STREAM_CFG_DEFAULT();

    i2sw_cfg.i2s_config.communication_format = I2S_COMM_FORMAT_I2S_MSB;

    i2sw_cfg.type = AUDIO_STREAM_WRITER;

    i2s_stream_writer = i2s_stream_init(&i2sw_cfg);



    ESP_LOGI(TAG, "[4.0] Init Element");

    audio_element_cfg_t cfg = DEFAULT_AUDIO_ELEMENT_CONFIG();

    cfg.open = el_open;

    cfg.process = el_process;

    cfg.tag = "my_el";



    my_el = audio_element_init(&cfg);



    ESP_LOGI(TAG, "[5.0] Create pipeline");

    audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG();

    pipeline = audio_pipeline_init(&pipeline_cfg);

    mem_assert(pipeline);



    audio_pipeline_register(pipeline, my_el, "my_el");

    audio_pipeline_register(pipeline, i2s_stream_reader, "i2sr");

    audio_pipeline_register(pipeline, i2s_stream_writer, "i2sw");



    const char *link_tag[3] = {"i2sr", "my_el", "i2sw"};

    audio_pipeline_link(pipeline, link_tag, 3);



    ESP_LOGI(TAG, "[6.0] Run pipeline");

    audio_pipeline_run(pipeline);



    ESP_LOGI(TAG, "[ * ] Starting ...");

    while (1) {

        vTaskDelay(2000);

    }

}

Here I can hear my voice on the left channel but on the right channel there is a very loud noise.

[paddlesteamer]

Ok, the problem is caused by `uint16_t`. Converting it to `int16_t` solved the problem. Here is the corrected part:

Code: Untitled.c Select all

static int el_process(audio_element_handle_t self, char *buf, int len) {

    int rsize = audio_element_input(self, buf, len);



    if (len != rsize || (rsize % 4) != 0) {

        ESP_LOGW(TAG, "unexpected rsize: %d, len: %d", rsize, len);

    } 



    int16_t lSample, rSample;

    char * lSamplep = (char *) &lSample;

    char * rSamplep = (char *) &rSample;

    for (int i =0; i<rsize; i+=4) {

        rSamplep[0] = buf[i];

        rSamplep[1] = buf[i+1];



        lSamplep[0] = buf[i+2];

        lSamplep[1] = buf[i+3];



        float a = convertToF32(rSample);

        a *= 0.5;

        rSample = convertToInt16(a);



        buf[i] = rSamplep[0];

        buf[i+1] = rSamplep[1];



        buf[i+2] = lSamplep[0];

        buf[i+3] = lSamplep[1];

    }



    rsize = audio_element_output(self, buf, rsize);



    return rsize;

}

Note that int16 to float conversion is not required, I will be using floats in my code that's what it's used here.

[mike84]

Ok, the problem is caused by `uint16_t`. Converting it to `int16_t` solved the problem. Here is the corrected part:

Code: Select all

static int el_process(audio_element_handle_t self, char *buf, int len) {
    int rsize = audio_element_input(self, buf, len);
 
    if (len != rsize || (rsize % 4) != 0) {
        ESP_LOGW(TAG, "unexpected rsize: %d, len: %d", rsize, len);
    }
 
    int16_t lSample, rSample;
    char * lSamplep = (char *) &lSample;
    char * rSamplep = (char *) &rSample;
    for (int i =0; i<rsize; i+=4) {
        rSamplep[0] = buf[i];
        rSamplep[1] = buf[i+1];
 
        lSamplep[0] = buf[i+2];
        lSamplep[1] = buf[i+3];
 
        float a = convertToF32(rSample);
        a *= 0.5;
        rSample = convertToInt16(a);
 
        buf[i] = rSamplep[0];
        buf[i+1] = rSamplep[1];
 
        buf[i+2] = lSamplep[0];
        buf[i+3] = lSamplep[1];
    }
 
    rsize = audio_element_output(self, buf, rsize);
 
    return rsize;
}

Note that int16 to float conversion is not required, I will be using floats in my code that's what it's used here.

Hello paddlesteamer!

I want to make an ESP32 based device. The input signal will be fed via the I2S interface (num_0) to the ESP32. It will be for example a regular mems microphone.

Next, I want to do audio processing in ESP32. Sound processing will be to encrypt it, for example AES (in the AES library I found, it is encrypted in groups of 16 elements). This library takes 2-3 ms to encrypt 16 elements. If the buffer has 64 elements, then it will take about 12 ms to encrypt.

After encryption, I want to output sound also on the I2S interface (num_0 or num_1).
In principle, I wrote this code, but when I start doing something between the functions i2s_read_bytes and i2s_write_bytes, then I start having problems with the sound. For the experiment, I just inserted delay 6 or more ms.
Here is my code:

Code: Select all

#include <driver/i2s.h> 
const int sample_rate = 16000; // 44100 
esp_err_t err;
 
uint16_t buf_len = 32; // 1024
uint8_t *buf = (uint8_t*) calloc(buf_len, sizeof(uint8_t));
int bytes_written = 0;
 
void setup() {
Serial.begin(500000);
delay(100);
Serial.println(" "); Serial.println("Setup I2S ...");
delay(500);
i2s_install();
i2s_setpin();
delay(500); 
}
 
void loop() {
int bytes_read = 0;
while(bytes_read == 0) {
   bytes_read = i2s_read_bytes(I2S_NUM_0, buf, buf_len, portMAX_DELAY);
 } 
 
delay (6); 
 
i2s_write_bytes(I2S_NUM_0, buf, bytes_read, portMAX_DELAY); 
} 
 
void i2s_install() {
const i2s_config_t i2s_config = {
.mode = i2s_mode_t(I2S_MODE_MASTER | I2S_MODE_TX | I2S_MODE_RX),
.sample_rate = sample_rate,
.bits_per_sample = i2s_bits_per_sample_t(16),
.channel_format = I2S_CHANNEL_FMT_ONLY_LEFT,
.communication_format = i2s_comm_format_t(I2S_COMM_FORMAT_I2S | I2S_COMM_FORMAT_I2S_MSB),
.intr_alloc_flags = ESP_INTR_FLAG_LEVEL1, // default interrupt priority
.dma_buf_count = 16, // 32
.dma_buf_len = 64, // 32 * 2
};
err = i2s_driver_install(I2S_NUM_0, &i2s_config, 0, NULL); 
if (err != ESP_OK) { 
 Serial.printf("Failed installing driver_1: %d\n", err);
  while (true);
 } 
Serial.println("I2S driver installed."); 
} 
 
void i2s_setpin() {
const i2s_pin_config_t pin_config = {
.bck_io_num =   26,
.ws_io_num =    25,
.data_out_num = 22,
.data_in_num =  33
  }; 
err = i2s_set_pin(I2S_NUM_0, &pin_config);
if (err != ESP_OK) {
Serial.printf("Failed setting pin_1: %d\n", err);
while (true);
}
Serial.println("I2S pins installed."); Serial.println(" ");
}

Processing takes 10-15 ms for buf_len = 64. For experiment, I try setting delay (12).
If you set delay (12) and buf_len = 1024, then everything is fine. If you set buf_len = 64, then the sound is wheezing at the DAC output.

But here's one thing. The larger the buffer (buf_len) in this case, the longer it will take to process and, accordingly, it will no longer be 10-15 ms.
Taking measurements (with millis), I got that 16 elements of the buffer are processed (by the aes encryption library) on average 3ms. Thus, it is impossible to set a large value of buf_len.

Can you tell me what to do next?

[paddlesteamer]

Hello,

I think your approach is not real-time. In the loop function, you first call `i2s_read_bytes` then wait 6 ms, then you send them to i2s output, and then, again you read the next batch. The latter batch you read is 6 ms late. I don't know Arduino-style coding on esp32 but there should be a method where you can register a callback function which will be called every time there are samples ready on i2s input. In your callback function, you should read bytes from i2s input to a buffer, forward them immediately to i2s output, and only after that you should run the encryption process on the buffer. If you really want to wait for AES encryption to complete before you output samples , then you should have an output buffer instead of outputting read bytes immediately, you should output bytes from the beginning of the output buffer and append the read bytes to the output buffer after the encryption process is finished.

[mike84]

Hello,

I think your approach is not real-time. In the loop function, you first call `i2s_read_bytes` then wait 6 ms, then you send them to i2s output, and then, again you read the next batch. The latter batch you read is 6 ms late. I don't know Arduino-style coding on esp32 but there should be a method where you can register a callback function which will be called every time there are samples ready on i2s input. In your callback function, you should read bytes from i2s input to a buffer, forward them immediately to i2s output, and only after that you should run the encryption process on the buffer. If you really want to wait for AES encryption to complete before you output samples , then you should have an output buffer instead of outputting read bytes immediately, you should output bytes from the beginning of the output buffer and append the read bytes to the output buffer after the encryption process is finished.

Paddlesteamer! Thank you very much for your reply!
Please, can you show at least a little example in the code how I need to do it?

So, I need to create one (or maybe several) intermediate buffers?

Can you add a few lines to my code to tell me how to do it?

[chitarrastonata]

Ciao paddlesteamer,
the easiest way to handle a signed value is to use an signed variable!
You are using unsigned, so it became everything complicated.
Try this sample code, it implement a very simple echo:

#define ECHO_LEN (1024*32)
int16_t echoBufR[ECHO_LEN];
int16_t echoBufL[ECHO_LEN];
uint16_t echoPtr = 0;

static int el_process(audio_element_handle_t self, char *buf, int len) {
int rsize = audio_element_input(self, buf, len);

if (len != rsize || (rsize % 4) != 0) {
ESP_LOGW(TAG, "unexpected rsize: %d, len: %d", rsize, len);
}

int16_t rSample, lSample;
for (int i =0; i<rsize; i+=4) {
rSample = *((int16_t*)(&buf[i+0]));
lSample = *((int16_t*)(&buf[i+2]));

// ECHO
rSample += (echoBufR[echoPtr] / 2);
lSample += (echoBufL[echoPtr] / 2);
echoBufR[echoPtr] = rSample;
echoBufL[echoPtr] = lSample;
if(++echoPtr >= ECHO_LEN) {
echoPtr = 0;
}

*((int16_t*)(&buf[i+0])) = rSample;
*((int16_t*)(&buf[i+2])) = lSample;
}

rsize = audio_element_output(self, buf, rsize);
return rsize;
}