Improve SD logger

[jevgienij]

I'm working on a project to read data over CAN, display it on LCD and log data to SD card. LCD is on HSPI and SD card on VSPI. CAN and LCD code is working on core 1 and SD on core 0, there's no wifi. It's all working pretty well but now I'm looking for further improvements. I set the log to 20 Hz and took a sample of 117024 lines. Below is the "uniq -c" output of the seconds that weren't logged at 20 Hz (left column is the actual rate and right column is a specific second in the log). So only 21 out of 117024 lines. But why does this happen and how can I make sure that every line is logged at 20 Hz?

• 19 1632
  18 3195
  17 5393
  16 5401
  16 1622
  14 5397
  14 1628
  14 1625
  14 1624
  13 5400
  13 5398
  13 5396
  13 5395
  13 5394
  13 1626
  12 1631
  12 1630
  12 1627
  12 1623
  8 5399
  8 1629

Code: SD-Logger.c Select all

float timeStamp;

char filenameBuffer[23];

String filename;

bool logStarted = 0;

uint32_t lastSyncTime = 0, logStartTime = 0;



File file;



void initSD()

{

	if (SD.begin(5, SPI, 80000000))

	{

		Serial.println("Card Mount OK");

		updateFromFS(SD);

		generateFileName();

		sdInitialised = true;

	}

	else

	{

		Serial.println("Card Mount Failed");

		return;

	}

}



void logToSD()

{

	if (!sdInitialised) return;

	if (!EMUfresh) return;



	if (!logStarted)

	{

		file = SD.open(filenameBuffer, FILE_APPEND);

		if (file)

		{

			file.print("Time,RPM,MAP,TPS,IAT,CLT,Lambda,PulseWidth,EGT1,EGT2,Vbatt,Baro,IgnAngle,LambdaCorr,Gear,Pwm1,Vss,EmuTemp,CEL,FLAGS1,LambdaTarget,OilTemp,OilPress,CanState\r\n");

			logStartTime = millis();

			logStarted = true;

		}

	}

	

	uint32_t duration = millis() - logStartTime;

	uint32_t seconds = duration / 1000;

	uint32_t milliseconds = duration % 1000;

	String dataString = "";

	

	dataString += seconds;

	dataString += ".";

	if (milliseconds < 100) { dataString += "0"; }

	if (milliseconds < 10) { dataString += "0"; }

	dataString += milliseconds;

	dataString += ",";

	dataString += String(emu_data.RPM);

	dataString += ",";

	dataString += String(emu_data.MAP);

	dataString += ",";

	dataString += String(emu_data.TPS);

	dataString += ",";

	dataString += String(emu_data.IAT);

	dataString += ",";

	dataString += String(emu_data.CLT);

	dataString += ",";

	dataString += String(emu_data.lambda);

	dataString += ",";

	dataString += String(emu_data.pulseWidth);

	dataString += ",";

	dataString += String(emu_data.EGT1);

	dataString += ",";

	dataString += String(emu_data.EGT2);

	dataString += ",";

	dataString += String(emu_data.vBat);

	dataString += ",";

	dataString += String(emu_data.baro);

	dataString += ",";

	dataString += String(emu_data.ignAngle, 0);

	dataString += ",";

	dataString += String(emu_data.lambdaCorr, 0);

	dataString += ",";

	dataString += String(emu_data.gear);

	dataString += ",";

	dataString += String(emu_data.pwm1);

	dataString += ",";

	dataString += String(emu_data.vssSpeed);

	dataString += ",";

	dataString += String(emu_data.emuTemp);

	dataString += ",";

	dataString += String(emu_data.cel);

	dataString += ",";

	dataString += String(emu_data.flags1);

	dataString += ",";

	dataString += String(emu_data.lambdaTarget);

	dataString += ",";

	dataString += String(emu_data.oilTemperature);

	dataString += ",";

	dataString += String(emu_data.oilPressure);

	//dataString += ",";

	//dataString += String(can_error_data.state[1]); //Only first character

	dataString += "\r\n";



	file.print(dataString.c_str());



	if (millis() - lastSyncTime > 1000)

	{

		lastSyncTime = millis();

		file.flush();

	}

}



void generateFileName()

{

	char intBuffer[5];

	

	strcpy(filenameBuffer, "/");

	itoa(random(1000, 9999), intBuffer, 10);

	strcat(filenameBuffer, intBuffer);

	strcat(filenameBuffer, "-");

	itoa(random(10), intBuffer, 10);

	strcat(filenameBuffer, intBuffer);

	strcat(filenameBuffer, "-");

	itoa(random(10), intBuffer, 10);

	strcat(filenameBuffer, intBuffer);

	strcat(filenameBuffer, "_");

	itoa(random(10), intBuffer, 10);

	strcat(filenameBuffer, intBuffer);

	strcat(filenameBuffer, ".");

	itoa(random(10), intBuffer, 10);

	strcat(filenameBuffer, intBuffer);

	strcat(filenameBuffer, ".");

	itoa(random(10), intBuffer, 10);

	strcat(filenameBuffer, intBuffer);

	strcat(filenameBuffer, ".csv");

	

	Serial.println(filenameBuffer);

}

[markdh102]

I do something very similar. I log GPS/MEMS/BME/Bluetooth OBD/Timestamp every 100mS and store to an SD card.
I found that most writes to the SD card took <5mS. Sometimes however they took approx 320mS.
This is due to the way the blocks are written to the SD card at a low level and can't be changed (at least I don't think so).
So, I created a circular RAM buffer to hold my records. This all happens at 10Hz on the main core.
In the 2nd processor core, I pull 1 record at a time out of this circular buffer and write it to the SD card. That way, it doesn't matter if it takes a while to write as I'm still storing records into my RAM buffer.
I no longer have timestamps other than 100ms intervals.
Hope this helps.

[jevgienij]

Can you show some example code of it?

[markdh102]

This is a very abbreviated form of the code I use. Hopefully it will show you what I am doing.
I don't know if this will compile. I've not got my system to hand. And it is MOST DEFINITELY NOT the most elegant of code.

Code: Select all

#include <Arduino.h>
#include "SdFat.h"


SdFat                   _sd;
File                    _myFile;


#define MAX_RECORDS                   100
#define MAX_COMBINED_OBD_DATA_LENGTH  200
#define MAX_RECORD_LENGTH             232 + MAX_COMBINED_OBD_DATA_LENGTH


uint32_t                _ulngRecordCount;
int volatile            _intInCount;
int volatile            _intOutCount;
char volatile           _bytDataArray[MAX_RECORDS][MAX_RECORD_LENGTH];
char                    _strTempSDcardData[MAX_RECORD_LENGTH];
char                    _strCombinedOBDdata[MAX_COMBINED_OBD_DATA_LENGTH];
bool volatile           _blnRecordLocked;
bool volatile           _blnStopLogging;
char                    _bytFileInUse;


int setupSDcard(void)
{
  uint32_t  cardSize;
  int       intSDcardStatus = 0;
  
  // Initialize at the highest speed supported by the board that is
  // not over 50 MHz. Try a lower speed if SPI errors occur.
  if (!_sd.begin(SS, SD_SCK_MHZ(20))) 
  {
    #ifdef SERIAL_DEBUG
    Serial.println(F("SD: Mount failed"));
    #endif
    return -1;
  }

  cardSize = _sd.card()->cardSize();
  if (cardSize == 0) 
  {
    #ifdef SERIAL_DEBUG
    Serial.println(F("SD: Cardsize failed"));
    #endif
    return -1;
  }
  
  switch (_sd.card()->type()) 
  {
    case SD_CARD_TYPE_SD1:
      #ifdef SERIAL_DEBUG
      Serial.println(F("SD: Type SD1"));
      #endif
    break;

    case SD_CARD_TYPE_SD2:
      #ifdef SERIAL_DEBUG
      Serial.println(F("SD: Type SD2"));
      #endif
    break;

    case SD_CARD_TYPE_SDHC:
      #ifdef SERIAL_DEBUG
      if (cardSize < 70000000) 
      {
        Serial.println(F("SD: Type SDHC"));
      } 
      else 
      {
        Serial.println(F("SD: Type SDXC"));
      }
      #endif
      break;
    
    default:
      intSDcardStatus = -2;      
  }
  
  if (intSDcardStatus == 0)
  {
    #ifdef SERIAL_DEBUG
    Serial.printf("SD: Size = %d BYTES\r\n", (uint32_t) (cardSize * 0.512));

    _sd.ls("/", LS_SIZE | LS_R );
    #endif
  }  
  return intSDcardStatus;
}


// This code runs on the 2nd processor core and handles writing to the SD card
void Task1Code(void *parameter)
{
  int   next;
  
  for (;;)
  {
    delay(1);
    
    // Only if the foreground loop() is not adding a new record
    if (_blnRecordLocked == false)
    {
      // If we are trying to stop logging then make sure we can't write any more records
      if (_blnStopLogging == true)
      {
        _blnRecordLocked = true;
      }
      
      // Run a circular buffer
      if (_intInCount != _intOutCount)
      {
        next = _intOutCount + 1;
        if (next >= MAX_RECORDS)
          next = 0;
        
        if (strlen((char *)&_bytDataArray[_intOutCount][0]) < MAX_RECORD_LENGTH)
          strcpy(_strTempSDcardData, (char *)&_bytDataArray[_intOutCount][0]);
        else
          strcpy(_strTempSDcardData, "TOO LONG\r\n");

        // This normally takes < 5mS, but occasionally (when it writes the buffer) it takes about 320mS !!!
        // This is why we run it in a seperate core so it can do it's own thing without holding anyone up
        _myFile.println(_strTempSDcardData);

        _intOutCount = next;
      }

      // The other core checks this flag when trying to stop logging
      // When we get here, the SD card has been fully written so allow the
      // foreground task on the other core to continue
      _blnStopLogging = false;
    }
  }
}


// I have read that the 'idle' task runs at priority 0 so create any other tasks with a higher one or else 'weirdness' may result???
// Setup() and Loop() run on core 1 at priority 1
// WiFi is handled by core 0 and runs at MAX_PRIORITY-1 (I think this equates to 24)
// Bluetooth is handled by core 0 and runs at priority 18
void createAtask(void)
{
  //                      Function   Name     Stack  I     P  Handle  Core
  // Stack is siize in words
  // I is task input parameter
  // P is priority 0 = lowest
  xTaskCreatePinnedToCore(Task1Code, "Task1", 10000, NULL, 1, &_task1, 0);  
}


void stopLogging(void)
{
  // Only if in use or paused
  if (_bytFileInUse == 'Y' || _bytFileInUse == 'P')
  {
    if (_sd.exists(_strFilename))
    {
      // Stop the process on the other core accessing the SD card
      _blnStopLogging = true;
      while (_blnStopLogging == true) ;

      _myFile.close();
      _bytFileInUse = 'N';
    }
  }  
}


void startLogging(void)
{
  // Only if not already logging
  if (_bytFileInUse == 'N')
  {
    // Reset the SD card writer engine
    _blnStopLogging = false;
    _intOutCount = _intInCount = 0;
    _blnRecordLocked = false; 
       
    _ulngRecordCount = 0;

		// NOTE I HAVE AN RTC DEVICE FITTED

    // We need to start a new file so update the date/time registers from the RTC
    //readDateTime();
    // We are limited to an 8.3 filename format so try to be as unique as we can...
    // YMMDDHHm.mss
    //sprintf(_strFilename, "%d%02d%02d%02d%d.%d%02d", getYears() % 10, getMon(), getDate(), getHour(), getMinute() / 10, getMinute() % 10, getSecond());
    // set date time callback function
    //SdFile::dateTimeCallback(dateTime);

		strcpy(_strFilename, "TEST.DAT");

    // Write my header data (I've removed the OBD calls)
    // O_WRITE | OCREAT is supposedly faster than FILE_WRITE
    _myFile = _sd.open(_strFilename, O_WRITE | O_CREAT);
    _myFile.print("Battery Volts : ");
    _myFile.print("OBD standards supported : ");
    _myFile.print("DTC codes : ");
    _myFile.print("VIN :");
    _myFile.print("Supported PIDS : ");
    _myFile.println("\r\n");
    
    _bytFileInUse = 'Y';
  }

  // If paused just continue logging
  if (_bytFileInUse == 'P')
  {
    _bytFileInUse = 'Y';
  }
}


void updateTheFile(void)
{
  char  strCompleteRecord[MAX_RECORD_LENGTH];
  
  if (SD.totalBytes() - SD.usedBytes() > 1000)
  {
    sprintf(strCompleteRecord, "ALL OF MY DATA"); 
    
    _ulngRecordCount++;
    
    if (strlen(strCompleteRecord) < MAX_RECORD_LENGTH)
    {
      int next;
      
      _blnRecordLocked = true;
      next = _intInCount + 1;
      if (next >= MAX_RECORDS)
      {
        next = 0;
      }
      if (next != _intOutCount)
      {
        strcpy((char *)&_bytDataArray[_intInCount][0], strCompleteRecord);
        _intInCount = next;
      }
      _blnRecordLocked = false;
    }
  }  
}


void setup() 
{
  _ulngRecordCount = 0;
  _bytFileInUse = 'N';
  _intOutCount = _intInCount = 0;
  _blnRecordLocked = false;
  _blnStopLogging = false;

  // Get the SD card task running
  createAtask();

  setupSDcard()
  
  startLogging();
}


void loop() 
{
  if (_bytFileInUse == 'Y')
  {
    // I call this every 100mS
    updateTheFile();
  }
}