Text file transfer between PC and Arduino using Batch scripts

Question

I have a personal project based on file transfer between an Arduino Due and my PC. Theses files are in text format and can go over 10Mb.

I firstly bought an SD SPI module to connect an external SD card to my Arduino and a compatible micro-SD card.

SD card sided, everything works well, the card is detected and data storage is also working.

To transfer data from my PC, I am using Batch scripts that send text file content over Serial port used by Arduino.

The problem comes: I am unable to transfer over 65535 length file over Serial port at once.

If the file contains more than 65535 characters, the transfer stuck and I am unable to do anything until I reset the Arduino card. I tried with a bit more characters, but it only writes firsts characters in the file.

So, if my file does 65635 characters, only the first 100 characters will be written in the new file.

But as I said, my goal is to transfer huge files.

To illustrate my problem I give you some codes on Arduino and Batch scripts.

Arduino part

I launch different states with buttons.

1. First one run a "send_file" batch script that will send data to serial port and write into a new file on SD card.
2. A second one will run a "read_serial" batch script that will read file content sent over Serial port by Arduino.
3. The third one is used to delete the file created before.

Note that I use RGB led for different states, and also Keyboard library to do actions on my computer.

Here is the complete code:

#include <Wire.h>
#include <Keyboard.h>
#include <SPI.h>
#include <SD.h>

File myFile;

const int chipSelect = 53;

String AltGrazerty = "~#{[|`\\^@#]}";
String shiftazerty = "QBCDEFGHIJKL?NOPARSTUVZXYW1234567890 Q+QQQQM%Q./Q>";
String azerty = "qbcdefghijkl,noparstuvzxyw&q\"'(-q_qq )=q$q*mqq;:!<";
const byte scancode[] = { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 100 };

int BP1 = 28, BP2 = 29, BP3 = 31, BP4 = 30, BP5 = 33, BP6 = 32;

int red_light_pin= 9;
int green_light_pin = 11;
int blue_light_pin = 10;

int ledstate = 0;

int val1 = 0, val2 = 0, val3 = 0, val4 = 0, val5 = 0, val6 = 0, flag = 0;

void setup() {
  Serial.begin(115200);
  Keyboard.begin();

  if (!SD.begin(chipSelect)) {
    while (1);
  }
  
  myFile = SD.open("test.txt");
  while (myFile.available()) {
          Serial.write(myFile.read());
        }
  myFile.close();
  Serial.flush();
      
  pinMode(BP1, INPUT_PULLUP);
  pinMode(BP2, INPUT_PULLUP);
  pinMode(BP3, INPUT_PULLUP);
  pinMode(BP4, INPUT_PULLUP);
  pinMode(BP5, INPUT_PULLUP);
  pinMode(BP6, INPUT_PULLUP);
  
  pinMode(red_light_pin, OUTPUT);
  pinMode(green_light_pin, OUTPUT);
  pinMode(blue_light_pin, OUTPUT);
}

void loop() {

  val1 = digitalRead(BP1);
  val2 = digitalRead(BP2);
  val3 = digitalRead(BP3);
  val4 = digitalRead(BP4);
  val5 = digitalRead(BP5);
  val6 = digitalRead(BP6);

  switch (ledstate){
    case 0 : RGB_color(255, 239, 0); 
    break;
    case 1 : RGB_color(255, 0, 0); 
    break;
    case 2 : RGB_color(0, 255, 0); 
    break;
    case 3 : RGB_color(0, 0, 255); 
    break;
  }

  if (val1 == 0) 
  {
    if (flag == 0) 
    {
      
      Keyboard.press(KEY_LEFT_GUI);
      Keyboard.press('r');
      Keyboard.releaseAll();
      delay(250);
      Keyfr("C:/Users/***/Desktop/send_data.bat");
      delay(500);
      Keyboard.press(KEY_RETURN);
      Keyboard.releaseAll();

      String text = Serial.readString();
      
      myFile = SD.open("test.txt",FILE_WRITE);

      if(myFile){
      myFile.println(text);
      myFile.close();
      } else {}

      ledstate = 1;
      flag = 1;
    }
  } else if (val2 == 0) {
    if (flag == 0) {

      Serial.flush();

      delay(500);

      Keyboard.press(KEY_LEFT_GUI);
      Keyboard.press('r');
      Keyboard.releaseAll();
      delay(250);
      Keyfr("C:/Users/***/Desktop/read_data.bat");
      delay(500);
      Keyboard.press(KEY_RETURN);
      Keyboard.releaseAll();
    
      ledstate = 2;
      flag = 1;
    }
  } else if (val3 == 0) {
    if (flag == 0) {

      Keyboard.press(KEY_LEFT_GUI);
      Keyboard.press('r');
      Keyboard.releaseAll();
      delay(250);
      Keyfr("C:/Users/***/Desktop/del.bat");
      delay(500);
      Keyboard.press(KEY_RETURN);
      Keyboard.releaseAll();
      
      myFile = SD.open("test.txt");
      if (myFile) {

         SD.remove("test.txt");

        myFile.close();
      } else {}

      ledstate = 3;
      flag = 1;
    }
  } else if (val4 == 0) {
    if (flag == 0) {
      
      flag = 1;
    }
  } else if (val5 == 0)
  {
    if (flag == 0) {
      
      flag = 1;
    }
  } else if (val6 == 0) {
    if (flag == 0) {

      flag = 1;
    }
  } else if (val1 == 1 && val2 == 1 && val3 == 1 && val4 == 1 && val5 == 1 && val6 == 1) 
  {
    flag = 0;
  }

  delay(50);
}

void Keyfr(const String &Texte) {  
  int j = -1;

  for (unsigned int i = 0; i < Texte.length(); i++) {
    char c = Texte.charAt(i);

    if (c == '\t') {
      Keyboard.write(KEY_TAB);
    }

    int index = azerty.indexOf(c);
    if (index > -1) {
      j = scancode[index] + 136;
      Keyboard.write(j);
    } else {
      index = shiftazerty.indexOf(c);
      if (index > -1) {
        j = scancode[index] + 136;
        Keyboard.press(KEY_LEFT_SHIFT);
        Keyboard.press(j);
        Keyboard.releaseAll();
      } else {
        index = AltGrazerty.indexOf(c);
        if (index > -1) {
          j = scancode[index + 27] + 136;
          Keyboard.press(KEY_LEFT_CTRL);
          Keyboard.press(KEY_LEFT_ALT);
          Keyboard.write(j);
          Keyboard.releaseAll();
          if (index == 0 || index == 7) {
            Keyboard.press(KEY_LEFT_CTRL);
            Keyboard.press(KEY_LEFT_ALT);
            Keyboard.write(j);
            Keyboard.releaseAll();
            Keyboard.write(KEY_BACKSPACE);
          }
        }
      }
    }
  }
}

void RGB_color(int red_light_value, int green_light_value, int blue_light_value)
 {
  analogWrite(red_light_pin, red_light_value);
  analogWrite(green_light_pin, green_light_value);
  analogWrite(blue_light_pin, blue_light_value);
}

Batch part

I give you in this part only send and read scripts, even if I think theses scripts are not a problem.

Send script :

@echo off
setlocal EnableDelayedExpansion

@echo mode COM5 BAUD=115200 PARITY=n DATA=8

set "cmd=findstr /R /N "^^" file.txt | find C/ ":""

@type file.txt > COM5

Read script :

@echo off
setlocal EnableDelayedExpansion

@mode COM5 BAUD=115200 PARITY=n DATA=8 DTR=ON

set "cmd=findstr /R /N "^^" file.txt | find /C ":""

@type COM5>file2.txt

I hope my problem is understandable and my English is correct (not my primary language).

Thanks in advance.

Romain.

Answer 1

Like what you did for Arduino->PC, when receiving the file (PC->Arduino), you must loop and write on SD card continuously, until you get a protocoled END (can be anything: BRK, EOF, timeout, etc.). You can't receive the whole file within a single string, unless it is shorter than the UART buffer capacity... So it's not surprising that you get a 64 kB limit - it's even quite huge for an UART, in fact: I worked with some UARTs that had only 16 BYTES of internal buffer... Meaning that at 115,200 bauds, you need to flush this buffer EVERY MILLISECOND to avoid a buffer overrun!

Currently, your loop() function is excessively complex, and worst, contains delay instructions and branches... Reading from an UART with a polling loop can be done, but in this case, don't do anything else. Or setup a reception through an interruption handler.

Transferring through an UART isn't the same as sending through TCP/IP. There is no automatic retransmission, no ACK, NOTHING. You don't even know the size of what you are currently receiving - that's why, through a custom protocol, you usually send the size as FIRST element of a transmission, and/or you use something like UDP with ACK - sending fixed-size chunk, waiting ACK from receiver to send another chunk, the last chunk being marked as "partial" through its header (you'll need to make such a header, too).