Trial to port Freematics SIM5360 to TinyGSM

[laurentvm]

Hi Volodymyr ,

I tried to port your code to SIM5360 for freematics purpose:
https://github.com/stanleyhuangyc/Freematics/blob/master/ESPRIT/sim5360test/sim5360test.ino.
https://github.com/stanleyhuangyc/Freematics/blob/master/firmware_v5/traccar_client_sim5360/traccar_client_sim5360.ino
https://github.com/stanleyhuangyc/Freematics/blob/master/libraries/FreematicsPlus/FreematicsNetwork.cpp

As I'm not really not experienced with hardware, I did what I can up to now but I am stuck at the virtual read and write and available() function of the client.

If you want have some fun, you can review and try to complete the code attached below.
Laurent

/**
* @file       TinyGsmClientSIM5360.h
* @author     Volodymyr Shymanskyy
* @license    LGPL-3.0
* @copyright  Copyright (c) 2016 Volodymyr Shymanskyy
* @date       Nov 2016
*/
#include <Arduino.h>
#ifndef TinyGsmClientSIM5360_h
#define TinyGsmClientSIM5360_h

//#define TINY_GSM_DEBUG Serial


#if !defined(TINY_GSM_RX_BUFFER)
#define TINY_GSM_RX_BUFFER 64
#endif

#define TINY_GSM_MUX_COUNT 1
#define PIN_SIM_POWER 27
#define PIN_SIM_UART_RXD 16
#define PIN_SIM_UART_TXD 17
#include <TinyGsmCommon.h>
extern HardwareSerial xbSerial;
#define GSM_NL "\r"
static const char GSM_OK[] TINY_GSM_PROGMEM = "OK" GSM_NL;
static const char GSM_ERROR[] TINY_GSM_PROGMEM = "ERROR" GSM_NL;
#define BEE_BAUDRATE 115200L
enum SimStatus {
    SIM_ERROR = 0,
    SIM_READY = 1,
    SIM_LOCKED = 2,
};

enum RegStatus {
    REG_UNREGISTERED = 0,
    REG_SEARCHING    = 2,
    REG_DENIED       = 3,
    REG_OK_HOME      = 1,
    REG_OK_ROAMING   = 5,
    REG_UNKNOWN      = 4,
};

enum TinyGSMDateTimeFormat {
    DATE_FULL = 0,
    DATE_TIME = 1,
    DATE_DATE = 2
};

#define HTTP_CONN_TIMEOUT 5000


typedef enum {
    HTTP_DISCONNECTED = 0,
    HTTP_CONNECTED,
    HTTP_SENT,
    HTTP_ERROR,
} HTTP_STATES;

class GsmClient5360;

class TinyGsmSim5360
{
    friend class GsmClient5360;
private:
    char m_buffer[384] = {0};
    void xbTogglePower()
    {
        digitalWrite(PIN_SIM_POWER, HIGH);
        delay(50);
        digitalWrite(PIN_SIM_POWER, LOW);
        delay(2000);
        digitalWrite(PIN_SIM_POWER, HIGH);
        delay(1000);
        digitalWrite(PIN_SIM_POWER, LOW);
    }
    int dumpLine(char* buffer, int len)
    {
        int bytesToDump = len >> 1;
        for (int i = 0; i < len; i++) {
            // find out first line end and discard the first line
            if (buffer[i] == '\r' || buffer[i] == '\n') {
                // go through all following \r or \n if any
                while (++i < len && (buffer[i] == '\r' || buffer[i] == '\n'));
                bytesToDump = i;
                break;
            }
        }
        memmove(buffer, buffer + bytesToDump, len - bytesToDump);
        return bytesToDump;
    }  
    
    void xbPurge()
    {
        xbSerial.flush();
    }
    
    bool xbBegin(unsigned long baudrate)
    {
        pinMode(PIN_SIM_POWER, OUTPUT);
        digitalWrite(PIN_SIM_POWER, HIGH);
        xbSerial.begin(BEE_BAUDRATE, SERIAL_8N1, PIN_SIM_UART_RXD, PIN_SIM_UART_TXD);
        return true;
    }
    
    void xbWrite(const char* cmd)
    {
        xbSerial.print(cmd);
    }

    int xbRead(char* buffer, int bufsize, unsigned int timeout)
    {
        int n = 0;
        uint32_t t = millis();
        do {
            while (xbSerial.available() && n < bufsize - 1) {
                buffer[n++] = xbSerial.read();
            }
        } while (millis() - t <= timeout);
        buffer[n] = 0;
        return n;
    }
    
    int xbReceive(char* buffer, int bufsize, unsigned int timeout, const char** expected, byte expectedCount)
    {
        int bytesRecv = 0;
        uint32_t t = millis();
        do {
            if (bytesRecv >= bufsize - 16) {
                bytesRecv -= dumpLine(buffer, bytesRecv);
            }
            int n = xbRead(buffer + bytesRecv, bufsize - bytesRecv - 1, 100);
            if (n > 0) {
                bytesRecv += n;
                buffer[bytesRecv] = 0;
                for (byte i = 0; i < expectedCount; i++) {
                    // match expected string(s)
                    if (expected[i] && strstr(buffer, expected[i])) return i + 1;
                }
            } else if (n == -1) {
                // an erroneous reading
                break;
            }
        } while (millis() - t < timeout);
        buffer[bytesRecv] = 0;
        return 0;
    } 
public:
    char udpIP[16] = {0};
    uint16_t udpPort;
    
    TinyGsmSim5360(Stream& _stream)/*: stream(_stream)*/
    {
        memset(sockets, 0, sizeof(sockets));
    }

    bool begin() {
        return init();
    }

    bool init() {
        xbBegin(BEE_BAUDRATE);
        for (byte n = 0; n < 10; n++) {
            // try turning on module
            xbTogglePower();
            delay(2000);
            xbPurge();
            for (byte m = 0; m < 3; m++) {
                if (sendCommand("AT\r")) {
                    return true;
                }
            }
        }
        return false;
    }
    void stop()
    {
        sendCommand("AT+CPOWD=1\r");
    }

    bool restart() {
        stop();
        delay(3000);
        return init();
    }

    /*
    * GPRS functions
    */
    bool gprsConnect(const char* apn, const char* user = NULL, const char* pwd = NULL) {
        uint32_t t = millis();
        bool success = false;
        if (!sendCommand("ATE0\r")) return false;
        bool only3G = true;
        bool roaming = true;
        if (only3G) sendCommand("AT+CNMP=14\r"); // use WCDMA only
        do {
            do {
                Serial.print('.');
                delay(500);
                success = sendCommand("AT+CPSI?\r", 1000, "Online");
                if (success) {
                    if (!strstr(m_buffer, "NO SERVICE"))
                    break;
                    success = false;
                }
                if (strstr(m_buffer, "Off")) break;
            } while (millis() - t < 30000);
            if (!success) break;

            t = millis();
            do {
                success = sendCommand("AT+CREG?\r", 5000, roaming ? "+CREG: 0,5" : "+CREG: 0,1");
            } while (!success && millis() - t < 30000);
            if (!success) break;

            do {
                success = sendCommand("AT+CGREG?\r",1000, roaming ? "+CGREG: 0,5" : "+CGREG: 0,1");
            } while (!success && millis() - t < 30000);
            if (!success) break;

            do {
                sprintf(m_buffer, "AT+CGSOCKCONT=1,\"IP\",\"%s\"\r", apn);
                success = sendCommand(m_buffer);
            } while (!success && millis() - t < 30000);
            if (!success) break;

            //sendCommand("AT+CSOCKAUTH=1,1,\"APN_PASSWORD\",\"APN_USERNAME\"\r");

            success = sendCommand("AT+CSOCKSETPN=1\r");
            if (!success) break;

            success = sendCommand("AT+CIPMODE=0\r");
            if (!success) break;

            sendCommand("AT+NETOPEN\r");
        } while(0);
        if (!success) Serial.println(m_buffer);
        return success;
    }

    bool isGprsConnected() {
        uint32_t t = millis();
        unsigned int timeout = 5000;
        bool success=false;
        do {
            success = sendCommand("AT+CGATT?\r", 3000, "+CGATT: 1");
        } while (!success && millis() - t < timeout);
        /*
        sprintf(m_buffer, "AT+CSTT=\"%s\"\r", apn);
        if (!sendCommand(m_buffer)) {
            return false;
        }
        sendCommand("AT+CIICR\r");
        */
        return success;
    }

    String getLocalIP() {
        uint32_t t = millis();
        do {
            if (sendCommand("AT+IPADDR\r", 3000, "\r\nOK\r\n", true)) {
                char *p = strstr(m_buffer, "+IPADDR:");
                if (p) {
                    char *ip = p + 9;
                    if (*ip != '0') {
                        char *q = strchr(ip, '\r');
                        if (q) *q = 0;
                        return ip;
                    }
                }
            }
            delay(500);
        } while (millis() - t < 15000);
        return "";
    }

    bool modemConnect(const char* host, uint16_t port, uint8_t mux, bool ssl = false) {
        /*         int rsp;
        sendAT(GF("+CIPSTART="), mux, ',', GF("\"TCP"), GF("\",\""), host, GF("\","), port);
        rsp = waitResponse(75000L,
        GF("CONNECT OK" GSM_NL),
        GF("CONNECT FAIL" GSM_NL),
        GF("ALREADY CONNECT" GSM_NL),
        GF("ERROR" GSM_NL),
        GF("CLOSE OK" GSM_NL)   // Happens when HTTPS handshake fails
        );
        return (1 == rsp); */
        String ip = queryIP(host);
        strncpy(udpIP, ip.c_str(), sizeof(udpIP) - 1);
        udpPort = port;
        sprintf(m_buffer, "AT+CIPSTART=\"%s\",%u,1\r", host, port);
        if (sendCommand(m_buffer, HTTP_CONN_TIMEOUT)) {
            //m_state = HTTP_CONNECTED;
            return true;
        } else {
            //m_state = HTTP_ERROR;
            return false;
        }
        
    }
    String queryIP(const char* host)
    {
        sprintf(m_buffer, "AT+CDNSGIP=\"%s\"\r", host);
        if (sendCommand(m_buffer, 10000)) {
            char *p = strstr(m_buffer, host);
            if (p) {
                p = strstr(p, ",\"");
                if (p) {
                    char *ip = p + 2;
                    p = strchr(ip, '\"');
                    if (p) *p = 0;
                    return ip;
                }
            }
        }
        return "";
    }
    int modemSend(const void* buff, size_t len, uint8_t mux) {
        /*         sendAT(GF("+CIPSEND="), mux, ',', len);
        if (waitResponse(GF(">")) != 1) {
            return 0;
        }
        stream.write((uint8_t*)buff, len);
        stream.flush();
        if (waitResponse(GF(GSM_NL "DATA ACCEPT:")) != 1) {
            return 0;
        }
        streamSkipUntil(','); // Skip mux
        return stream.readStringUntil('\n').toInt(); */
        
        sprintf(m_buffer, "AT+CIPSEND=0,%u,\"%s\",%u\r", len, udpIP, udpPort);
        if (sendCommand(m_buffer, 100, ">")) {
            xbWrite((const char*)buff);
            return sendCommand(0, 1000);
        }
        return false;
    }
    char* receive(int* pbytes, unsigned int timeout)
    {
        char *data = checkIncoming(pbytes);
        if (data) return data;
        if (sendCommand(0, timeout, "+IPD")) {
            return checkIncoming(pbytes);
        }
        return 0;
    }

    char* checkIncoming(int* pbytes)
    {
        char *p = strstr(m_buffer, "+IPD");
        if (p) {
            *p = '-'; // mark this datagram as checked
            int len = atoi(p + 4);
            if (pbytes) *pbytes = len;
            p = strchr(p, '\n');
            if (p) {
                *(++p + len) = 0;
                return p;
            }
        }
        return 0;
    }
    size_t modemRead(size_t size, uint8_t mux) {

        /*
        sendAT(GF("+CIPRXGET=2,"), mux, ',', size);
        if (waitResponse(GF("+CIPRXGET:")) != 1) {
            return 0;
        }
        streamSkipUntil(','); // Skip mode 2/3
        streamSkipUntil(','); // Skip mux
        size_t len = stream.readStringUntil(',').toInt();
        sockets[mux]->sock_available = stream.readStringUntil('\n').toInt();

        for (size_t i=0; i<len; i++) {
            while (!stream.available()) {  }
            char c = stream.read();
            sockets[mux]->rx.put(c);
        }
        waitResponse();
        return len;
        */
        return 1;
    }

public:
    bool sendCommand(const char* cmd, unsigned int timeout = 2000, const char* expected = "\r\nOK\r\n", bool terminated = false)
    {
        if (cmd) {
            xbWrite(cmd);
        }
        m_buffer[0] = 0;
        byte ret = xbReceive(m_buffer, sizeof(m_buffer), timeout, &expected, 1);
        if (ret) {
            if (terminated) {
                char *p = strstr(m_buffer, expected);
                if (p) *p = 0;
            }
            return true;
        } else {
            return false;
        }
    }    
    
public:
    //Stream&       stream;
    GsmClient5360*    sockets[TINY_GSM_MUX_COUNT];
};


class GsmClient5360 : public Client
{
    friend class TinyGsmSim5360;
    typedef TinyGsmFifo<uint8_t, TINY_GSM_RX_BUFFER> RxFifo;

public:
    GsmClient5360() {}

    GsmClient5360(TinyGsmSim5360& modem, uint8_t mux = 1) {
        init(&modem, mux);
    }

    bool init(TinyGsmSim5360* modem, uint8_t mux = 1) {
        this->at = modem;
        this->mux = mux;
        sock_available = 0;
        prev_check = 0;
        sock_connected = false;
        got_data = false;
        at->sockets[mux] = this;
        return true;
    }

    virtual int connect(const char *host, uint16_t port) {
        stop();
        
        rx.clear();
        sock_connected = at->modemConnect(host, port, mux);
        return sock_connected;
    }

    virtual int connect(IPAddress ip, uint16_t port) {
        String host; host.reserve(16);
        host += ip[0];
        host += ".";
        host += ip[1];
        host += ".";
        host += ip[2];
        host += ".";
        host += ip[3];
        return connect(host.c_str(), port);
    }

    virtual void stop() {
        at->stop();
        sock_connected = false;
        //at->waitResponse();
        //rx.clear();
    }

    virtual size_t write(const uint8_t *buf, size_t size) {
        return at->modemSend(buf, size, mux);
    }

    virtual size_t write(uint8_t c) {
        return write(&c, 1);
    }

    virtual size_t write(const char *str) {
        if (str == NULL) return 0;
        return write((const uint8_t *)str, strlen(str));
    }

    virtual int available() {
        
        if (!rx.size() && sock_connected) {
            // Workaround: sometimes SIM5360 forgets to notify about data arrival.
            // TODO: Currently we ping the module periodically,
            // but maybe there's a better indicator that we need to poll
            if (millis() - prev_check > 500) {
                got_data = true;
                prev_check = millis();
            }
            //at->maintain();
        }
        return rx.size() + sock_available;
    }

    virtual int read(uint8_t *buf, size_t size) {
        
        //at->maintain();
        size_t cnt = 0;
        while (cnt < size && sock_connected) {
            size_t chunk = TinyGsmMin(size-cnt, rx.size());
            if (chunk > 0) {
                rx.get(buf, chunk);
                buf += chunk;
                cnt += chunk;
                continue;
            }
            // TODO: Read directly into user buffer?
            //at->maintain();
            if (sock_available > 0) {
                at->modemRead(rx.free(), mux);
            } else {
                break;
            }
        }
        return cnt;
    }

    virtual int read() {
        uint8_t c;
        if (read(&c, 1) == 1) {
            return c;
        }
        return -1;
    }

    virtual int peek() { return -1; } //TODO
    virtual void flush() { at->xbPurge(); }

    virtual uint8_t connected() {
        if (available()) {
            return true;
        }
        return sock_connected;
    }
    virtual operator bool() { return connected(); }

private:
    TinyGsmSim5360* at;
    uint8_t        mux;
    uint16_t       sock_available;
    uint32_t       prev_check;
    bool           sock_connected;
    bool           got_data;
    RxFifo         rx;
};
#endif

[vshymanskyy]

I'll leave this open until I get to implementing this modem. Thank you!

[8bit-bruno]

Hi,
i'm interested in the port to Freematics SIM5360.
Are you still working on this?

Thanks!

[eabase]

Hi @vshymanskyy @8bit-bruno @laurentvm

Did you ever get this to work?
Please send a PR and code example, if you did.
I'm also looking to make this work.

[eabase]

I added the code from above here, for easy review.

[eabase]

@vshymanskyy Can you please have another look at this? I have several of these modules and can test whatever you like on a daily basis.

[SRGDamia1]

I started skimming through the code above to see if I could smooth it in, but it has some odd customizations for the xb that it's written for and has some structural differences from how the others are written that make it difficult to proof. I'm not sure if it would even be faster to try and refactor it or to start with the SIM800 code and comb the manual for which AT commands changed.

[eabase]

I started to port it, first based on the SIM800, then on SIM7000 (which seem meaningless as there are several types in the 7000 series). So I ended up taking bits and pieces from all three solutions trying to patch together something. (The 3 pieces being SIM5360 from above, SIM800 and SIM7000 header files.)

At the end of the day I was trying to get the OTA over GSM working. Lot's of mods are needed, since all these devices are very different. Simply speaking, there are many AT command that doesn't have an equivalent in each.

And the way this library is written, doesn't make things more transparent...

---

I.e. @vshymanskyy why on earth are you writing all these complicated compiler definines, instead of putting stuff in *.cpp or header files?

// Utility templates for writing/skipping characters on a stream
#define TINY_GSM_MODEM_STREAM_UTILITIES() \
  template<typename T> \
  void streamWrite(T last) { \
    stream.print(last); \
  } \
  \
  template<typename T, typename... Args> \
  void streamWrite(T head, Args... tail) { \
    stream.print(head); \
    streamWrite(tail...); \
  } \
  \
  template<typename... Args> \
  void sendAT(Args... cmd) { \
    streamWrite("AT", cmd..., GSM_NL); \
    stream.flush(); \
    TINY_GSM_YIELD(); \
    /* DBG("### AT:", cmd...); */ \
  } \
  \
  bool streamSkipUntil(const char c, const unsigned long timeout_ms = 1000L) { \
    unsigned long startMillis = millis(); \
    while (millis() - startMillis < timeout_ms) { \
      while (millis() - startMillis < timeout_ms && !stream.available()) { \
        TINY_GSM_YIELD(); \
      } \
      if (stream.read() == c) { \
        return true; \
      } \
    } \
    return false; \
  }

• What is the purpose of the TINY_GSM_YIELD, defined to be delay(0) by default??
• Why is your TINY_GSM_DEBUG so convoluted? It's nearly impossible to get any debug output using this.
• Is there a reason why this buffer is so tiny? #define TINY_GSM_RX_BUFFER 64
• What is the purpose and use for TINY_GSM_USE_HEX?

This seem a like a promising project, but without more documentation it nearly impossible to contribute and improve.

[SRGDamia1]

Sorry! Some of the complexity is from @vshymanskyy and a bunch is from me. Documentation is hard.

All the complex defines are in the common file to avoid re-writing code while at the same time keeping the library small. The templates are for similar reasons. Using proper virtual classes would be (much) more readable and easier to maintain, but the size of the vtables is not zero and Tiny is an objective (See #280).

TINY_GSM_YIELD runs between each character to make sure nothing new characters come in while parsing a URC. If that were to happen you might end up splicing the URC's and getting a two unhandled halves of it instead. You shouldn't need it (delay(0)) unless you have a fast processor talking at a slow baud rate.

TINY_GSM_DEBUG use and sendAT use template to combine a bunch of thing and print them with a time stamp so you can use one DBG(this, "that", and, "some other thing") instead of a bunch of print statements every time. The definition looks confusing, but it shouldn't be hard to use it

Buffer size - yup, tiny. You can use build flags to make it as big as you want. On the modules with on-board buffering, there's not really an advantage to a bigger buffer. The cellular chip's buffer is probably plenty big. On the modules without internal buffering (A6/ESP8266/Neoway M590, I think) you need to make your buffer big enough to capture the largest response you're expecting to get.

The SIM800 and SIM7000 have the option of giving back characters from the buffer in HEX instead of ASCII. I'm not sure why you'd use it, but it's an option.

[SRGDamia1]

Alright, here's something to try: https://github.com/EnviroDIY/TinyGSM

No promises. It's created starting with the SIM7000 and just searching the AT manual to dump in commands where they seem to diverge. Test it with the examples and see if anything happens.

[eabase]

@SRGDamia1
Thanks! 😄 That look promising and similar to the modification I did myself.
BTW. I am using a Freematics Esprit (ESP32) with the GSM module.

A few things/questions though:

1. How did you determine TINY_GSM_MUX_COUNT?

2. I'm not sure the cellular chip buffer is plenty big. What does that mean? For example, it seem that the maximum send request length (AT+CHTTPSSEND=?) from an AT command is 4096 bytes. So I would assume the "buffer" would be something like that ~4k. Which (I guess) is why it is tricky to parse/receive files larger than that using HTTP, FTP or some streams, in one go. (I.e. not waiting for chunks.) I still don't understand clearly to how to deal with that, when using GSM/GPRS only. All this magic seem automatic when using WiFi, but totally different when using 2/3/4G.

3. Why is there a at->sendAT(GF("+CIPCLOSE="), mux); in the stop() function? It seem to break things...but perhaps other things are already broken.

4. Ok, so how to determine if I need to set a TINY_GSM_YIELD?

5. What is the intended function of restart()?
   Because it seem that setting CFUN=0, put the device into energy saving mode disabling everything, while the subsequent command CFUN=1,1 ensures a full module reset after changing back to full mode. This caused a boot loop in another experiment I did. I'm not sure any of this is needed for this device. If you actually do intend a full reset function, there there are commands for that, such as: AT+CRESET and AT+REBOOT, otherwise if just for some settings ATZ should work.

6. radioOff() should be +CFUN=4 not 0.

7. poweroff() should probably be +CPOF only, without the 1.

[eabase]

I'm getting an immediate boot loop when using this with the FileDownload example. Any idea what could be causing this?

[SRGDamia1]

1 - TINY_GSM_MUX_COUNT - The 5360 AT manual (v0.25, 16.20 AT+CIPOPEN) says there can be up to 10 simultaneous connections

2 - The TCP Rx buffer is 1500 bytes (see AT manual 16.35 AT+CIPRXGET). This library doesn't use any of the HTTP/HTTPS AT commands, it just opens the TCP client. I've honestly never tried to send/receive anything bigger.

3 - AT+CIPCLOSE is the command the manual gives for closing a socket in multi-socket mode.

4 - If you're getting spliced URC's, set the yield. ie, your debug log shows: ### Unhandled: +CIPR and then ### Unhandled: XGET: 2, #, # instead of accurately receiving data. I had the wrong URC in there for a socket closing, fixed.

5 - Reboot sounds great. All I did was start with the 7000 and search the manual for the command. If the same command existed, I left it there. I didn't take the time to look if it was the best way of doing it.

6 - Fixed

7 - Fixed

[SRGDamia1]

Before debugging the file download, how about posting some AT logs and getting the AllFunctions example working.

[eabase]

@SRGDamia1 Awesome info there! The manual is so obscure and super easy to miss important details.

• Also, I have not been able to get any of the ### Unhandled: debug logs. What are the settings supposed to be? Is it on by default?
• I've tracked down my boot loop to be a problem already here:

  // Set GSM module baud rate
  SerialAT.begin(115200);
  delay(3000);

Perhaps because I don't fully understand the comments here:

// Set serial for debug console (to the Serial Monitor, default speed 115200)
#define SerialMon Serial

// Set serial for AT commands (to the module)
// Use Hardware Serial on Mega, Leonardo, Micro
#define SerialAT Serial1

// or Software Serial on Uno, Nano
//#include <SoftwareSerial.h>
//SoftwareSerial SerialAT(2, 3); // RX, TX

// Increase RX buffer to capture the entire response
// Chips without internal buffering (A6/A7, ESP8266, M590)
// need enough space in the buffer for the entire response
// else data will be lost (and the http library will fail).
#define TINY_GSM_RX_BUFFER 1024

// See all AT commands, if wanted
//#define DUMP_AT_COMMANDS

// Define the serial console for debug prints, if needed
#define TINY_GSM_DEBUG SerialMon
//#define LOGGING  // <- Logging is for the HTTP library

• What is SerialMon and SerialAT? (I.e. which UART are Serial and Serial1 pointing to?)
• What libraries is this dependent on? (Perhaps I\m missing some?) So far I have found:
  • CRC32.h
  • ArduinoHttpClient
  • StreamDebugger
  • TinyGSM
  • Any others?

I'll have a look at AllFunctions now, but I see you just pushed some changes to your other repo...

[SRGDamia1]

I think that should be all the dependent libraries.

Are you seeing any output at all when you're running the program? I think all of the examples have the serial baud set at 115200; make sure that's what your serial port monitor is set to.

#define SerialMon is the UART going to your USB. I'm skimming the Freematics libraries and I think you should be using Serial.

SerialAT is the UART talking to the UART of the SIM5620. I think you'll need something like this:

HardwareSerial xbSerial(1);
#define SerialAT xbSerial

Do uncomment #define DUMP_AT_COMMANDS to get all the AT traffic (via StreamDebugger) on your Serial port monitor.

[eabase]

Yeah, I'm trying to figure out where all your serial is going...
So which is which of these?

  // Set your reset, enable, power pins here
  pinMode(20, OUTPUT);
  digitalWrite(20, HIGH);

  pinMode(23, OUTPUT);
  digitalWrite(23, HIGH);

Because on the Freematics there is no reset pin. So what you have up there is weird, and possibly dangerous, in case it is already connected to something else for output. in a different instance we have this:

#define PIN_XBEE_PWR 27
SerialAT.begin(115200, SERIAL_8N1, 16, 17);         // ESP: 16=UART#1 RX,  17=UART#1 TX

[eabase]

Ok, I just tried the AllFunctions.ino with the following results:

• The TinyGsmAutoBaud is not working on and need to be commented out, otherwise you get bootloops... This device uses 115200 for everything by default, so no need to mess with this.

• The code is using an empty string "" for SIM PIN, which is simply wrong behaviour if your SIM doesn't require any PIN. (It gives an error and eventually start asking for PUK2, since entering a PIN when none is required, is equivalent to entering the wrong PIN.

AT+CPIN?
+CPIN: READY
AT+CPIN=""

change the relevant line to this:

// EDIT HERE:  Set your SIM PIN (within quotes) "nnnn", if any, otherwise leave as 0.
#define GSM_PIN 0

• Although I have specifically set not to use USSD, functions for tests, it does it anyway...

#define TINY_GSM_TEST_GPRS true
#define TINY_GSM_TEST_WIFI false
#define TINY_GSM_TEST_CALL false
#define TINY_GSM_TEST_SMS false
#define TINY_GSM_TEST_USSD false
#define TINY_GSM_TEST_BATTERY false

• The Freematics has a very specific turn on/boot up sequence that need to finish, otherwise you will get unsoliciteted garbage from the bootup messages. The way to do it is:

#define PIN_XBEE_PWR 27
bool sim_power_on() {
    // The Freematics Esprit (SIM5360E) has a specific boot-up sequence 
    // that need to toggle power and wait for some setups.
    Serial.printf("Waiting for PB DONE...\r\n");
    pinMode(PIN_XBEE_PWR, OUTPUT);
    digitalWrite(PIN_XBEE_PWR, HIGH);
    delay(750);
    digitalWrite(PIN_XBEE_PWR, LOW);
    delay(2500);
    //xbPurge();  // Discard any stale data  <-- but depends: FreematicsPlus.h 
    digitalWrite(PIN_XBEE_PWR, HIGH);
    delay(500);
    digitalWrite(PIN_XBEE_PWR, LOW);
    Serial.printf(".");
    delay(10000); // Wait sufficiently long time for Modem to send last bootup message "PB DONE".
    return true;
}
...
void setup() {
  // Set console baud rate
  SerialMon.begin(115200);
  delay(100);
  sim_power_on();
  ...

[eabase]

The Bootup Sequence Look like this:

START

+CPIN: READY

OPL UPDATING

PNN UPDATING

SMS DONE

CALL READY

PB DONE

...and has to complete in order to ensure proper functionality.
This is also true for any subsequent reset operation.

[SRGDamia1]

Alright, so copy that sim_power_on() function into your example and run it in place of the other pin mode changes.

The examples are meant to be generic. You need to figure out the pins and stuff on your own. That's not the job of this library.

[SRGDamia1]

By the way that turn on sequence is using the PWR_KEY on the SIM5620.

[eabase]

Here is the current output log.

[eabase]

To fix USSD, need to replace with this:

#if TINY_GSM_TEST_USSD
  String ussd_balance = modem.sendUSSD("*111#");
  DBG("Balance (USSD):", ussd_balance);

  String ussd_phone_num = modem.sendUSSD("*161#");
  DBG("Phone number (USSD):", ussd_phone_num);
#endif
#endif

[SRGDamia1]

It's not connecting to the network. It warns that NETOPEN isn't preferred but doesn't say what actually is. Let's try using the PDP commands. Update and try again.

[eabase]

• Small bug here CIPTIMEOUT, need an = sign like this:

    sendAT(GF("+CIPTIMEOUT="), 75000, ',', 15000, ',', 15000);

[eabase]

I added the following to the header file, but it's still unhappy giving me an IP.

        // Define the PDP context
        sendAT(GF("+CGDCONT=1,\"IP\",\""), apn, '"');
        waitResponse();

        // Activate the PDP profile/context 1
        sendAT(GF("+CGACT=1,1")); // AT+CGACT=1
        //waitResponse(60000L);
        if (waitResponse(60000L) != 1)  // was 150,000
            return false;

        // Attach to GPRS Packet domain)
        sendAT(GF("+CGATT=1"));
        if (waitResponse(60000L) != 1)
            return false;

[SRGDamia1]

Update from my fork and post the log please.

Remember, I don't have any of the boards or chips you're working with. I can't help you if you're not posting the AT responses.

[SRGDamia1]

I made some changes based on the TCP guide for the 7500/7600. Can you try both the AllFunctions and HttpClient examples and post the logs for both.

[eabase]

Strangely enough, with only these 4 lines, I can get an IP after cold boot.

# Check what's already stored:
AT+CGSOCKCONT?
AT+CSOCKAUTH?
AT+CGDCONT?

#AT+CGSOCKCONT=1,"IP","internet.xxxx.de","0.0.0.0",0,0
AT+CSOCKAUTH=1,1,"xxxx","xxxx"
AT+CGDCONT=1,"IP","internet.xxxx.de","0.0.0.0",0,0
AT+NETOPEN
AT+IPADDR

So I have no idea why we're trying to hop through all those hoops.
Something does remain in NAND flash after first setup, but what?

• Also there is a typo here : CIOPEN should be CIPOPEN.

• This look like a typo!

• modemGetConnected look weird, and should probably have or use the CNETSTART instead to check for open connections. (and elsewhere).

• The response for a working NETOPEN has to be +NETOPEN: 1,0 for an open/activated state. (If its just one digit, that's an error I think.)

It's way after my social hour, so the above proposed testing have to wait until tomorrow.

[SRGDamia1]

Typos fixed.

CNETSTART is for the state of the PDP connection, not of the individual TCP socket connections. The modemGetConnected function is a helper for the client to ask the modem if the socket is connected, not to check if the modem itself is connected to the network.

The AT log you posted had +NETOPEN: 1 with no commas. The problem was that it said OK first so the library moved on.

[eabase]

I'm not able to get the HttpClient to compile.

HttpClient:99:12: error: cannot declare variable 'http' to be of abstract type 'HttpClient'

 HttpClient http(client, server, port);

            ^

In file included from C:\Users\xxxx\Documents\Arduino\libraries\ArduinoHttpClient\src/ArduinoHttpClient.h:8:0,

from C:\Users\xxxx\Documents\Arduino\libraries\TinyGSM\examples\HttpClient\HttpClient.ino:88:

C:\Users\xxxx\Documents\Arduino\libraries\ArduinoHttpClient\src/HttpClient.h:41:7: 
note:   because the following virtual functions are pure within 'HttpClient':

 class HttpClient : public Client

The lines in question are:

TinyGsmClient client(modem);
HttpClient http(client, server, port);

[eabase]

@SRGDamia1 The compile problem above, seem similar to #283... Any ideas?

[SRGDamia1]

@eabase - as is mentioned in #238 and a few other issues, you can't use the most current ESP32 (1.0.2) core with the HttpClient library. That is a problem with the ESP32 core, not with TinyGSM.* Please complain on the issue over there: espressif/arduino-esp32#2755. Unfortunately, it doesn't look like it's going to be fixed for 1.0.3 either.

I forgot about that issue when I suggested using the HttpClient example. Try the WebClient examples if you want to keep using the current ESP Core.

[eabase]

• You are doing this:

  bool isGprsConnected() {
    sendAT(GF("+NETOPEN?"));
    if (waitResponse(GF(GSM_NL "+NETOPEN:")) != 1) {
      return false;
    }
    int res = stream.readStringUntil('\n').toInt();
    waitResponse();
    if (res != 1)
      return false;

But perhaps we should be doing something like this, instead?
(I tried but since I never get to a good connection, I can't test this.)

    bool isGprsConnected() {
        // The response should be: "+NETOPEN: 1,0" for an open/activated state
        sendAT(GF("+NETOPEN?"));
        //if (waitResponse(75000L, GF(GSM_NL "+NETOPEN: 1")) != 1) {          // QQQ with a "," or not??
        //if (waitResponse(75000L, GF(GSM_NL "+NETOPEN: 1,")) != 1) {         // QQQ with a "," or not??
        if (waitResponse(75000L, GF(GSM_NL "+NETOPEN: 1,0")) != 1) {          // QQQ with a "," or not??
            return false;
        }

and perhaps also wait even longer? Like ~120 s?

• Also, the manual say to close all open TCP/UDP sockets before using NETCLOSE, so perhaps we need to do it like this? :

    bool gprsDisconnect() {
        // Before NETCLOSE we need to close all open sockets with:
        // AT+CIPCLOSE      Close TCP or UDP socket
        // AT+CIPCLOSE=<link_num>
        // ToDo: iterate?
        sendAT(GF("+CIPCLOSE=1"));          // Close <link_num> socket
        if (waitResponse(60000L) != 1)
            return false;

        sendAT(GF("+NETCLOSE"));            // Close the network (NOTE: ALL sockets should be closed first)
        if (waitResponse(60000L) != 1)
            return false;

[eabase]

Could someone also explain the waitResponse() function. It's very convoluted and hard to follow what it actually returns.

[SRGDamia1]

Manual says there will only be anything after the first 1 in the NETOPEN response if it isn't open, so just go as far as the 1.

I suppose it doesn't hurt to close the sockets cleanly.

[eabase]

• Seem that the logic for your NETOPEN response is backwards, the manual states (in my words):

// For a NETOPEN? :   The response should be: "+NETOPEN: 1,0" for an open/activated state
// For a NETOPEN  :   The response should be: "+NETOPEN: 0" for successful open command 

[SRGDamia1]

Fixed

[SRGDamia1]

WaitResponse is listening to the modem. If you give it a string to look for, it will continue to collect text from the modem until it gets to a string that ends with that text. If don't specify how long to search, it defaults to 1s. If you don't specify the text, it defaults to OK/ERROR. While it's listening for expected responses, it also keeps an ear out for URC (unsolicited response codes). If it gets a URC, it parses it appropriately - notifying a socket of new data or remoste closing, etc.

[eabase]

I heavily reduced all the connection AT commands to only use the ones working from AT command line (as posted above), so now I finally got an IP! 🎉

It seem that the key was use a complete: AT+CGDCONT=1,"IP","internet.xxxx.de","0.0.0.0",0,0 including the 0.0.0.0 PDN IP. However, now I keep getting errors with CIPCLOSE.

According to AT Command manual, the responses to: AT+CIPCLOSE=<link_num> are:

1. GOOD

OK 
+CIPCLOSE: <link_num>,<err>

2. GOOD

+CIPCLOSE: <link_num>,<err>
OK 

3. BAD:

+CIPCLOSE: <link_num>,<err>
ERROR
ERROR

The AT sent is AT+CIPCLOSE=1, but the response is:

+CIPCLOSE: 1,4

ERROR

[eabase]

PS. We're using CIPCLOSE= in both gprsConnect() and in stop().

[eabase]

We may need to fix the NETCLOSE to look for return value 2, which is when the connection was never open in the first place. It seem that the subsequent commands fails due to buffer garbage when that happens. See page 418 for error codes.

[SRGDamia1]

Garbage in the buffer shouldn't be a problem. waitResponse is only looking at the end of the string.

[SRGDamia1]

But, if the chip says OK and then changes its mind and returns an error, that is a problem. Changed.

[SRGDamia1]

The CIPCLOSE should be ok with any of those responses.

[eabase]

I'll have to wait until Monday to continue these tests. But FYI:

• To get temperature, use:

# Enable Temparature Reading:
AT+CMTE=1
AT+CMTE?

• To get correct date/time from mobile network, use:

#---------------------------------------
# Set HTTP Time Synchronization Server
#---------------------------------------
AT+CHTPSERV?

AT+CHTPSERV="ADD","www.google.com",80,1
AT+CHTPSERV="ADD","www.google.com",443,1
AT+CHTPUPDATE                                // This need some time to complete...

AT+CCLK?

[eabase]

@SRGDamia1
Sara, thank you so much for helping to bring this code and module into life! ❤️

As of today FileDownload.ino and a customized gsm only OTA, are both working using my updated and modified TinyGsmClientSIM5360.h code. I have made some readability improvements to the code formatting and also added a lot of comments. Some of the connect code that you added, but that seem not to be needed, has also been commented out. So feel free to add, augment and improve. Thanks again, and enjoy. 🥇

• However, the work is not finished since there need to be better parsing of URC's in the waitResponse() code, in case of errors. These often interfere with other parts of the code parsing, when they suddenly appear out of the blue. Particularly troubling are the lost connections due to poor network coverage etc. Some of the more important URC's to parse correctly are:

# From page 468 in AT command Manual

+CIPEVENT: NETWORK CLOSED UNEXPECTEDLY
Network is closed for network error(Out of service, etc). When this event happens, 
user application needs to check and close all opened sockets, and then use AT+NETCLOSE 
to release the network library if AT+NETOPEN? shows the network library is still opened.

+IPCLOSE: <client_index>, <close_reason>
Socket is closed passively. 
<client_index>: a numeric parameter that identifies a connection. 
    The range of permitted values is 0 to 9. 
<close_reason>: a numeric parameter that identifies the reason to close a client: 
    0– close connection forwardly 
    1– closed connection passively 
    2– reset connection because of timeout of sending data

+CLIENT: < link_num >,<server_index>,<client_IP>:<port>
TCP server accepted a new socket client, 
    the index is <link_num>, 
    the TCP server index is <server_index>. 
    The peer IP address is <client_IP>, 
    the peer port is <port>.
 
----------------------------------------
Unsolicited TCP/IP command <err> Codes
----------------------------------------
0 operation succeeded
1 Network failure
2 Network not opened
3 Wrong parameter
4 Operation not supported
5 Failed to create socket
6 Failed to bind socket
7 TCP server is already listening
8 Busy
9 Sockets opened
10 Timeout
11 DNS parse failed
255 Unknown error
----------------------------------------

and perhaps of less importance:

----------------------------------------
The ones relevant for Receiving Data (p.414):
----------------------------------------
+CIPRXGET:
    //          1. If <mode> = 0 or 1:      OK
    //          2. If <mode> = 2 or 3:
    //                          a. If single-client:    +CIPRXGET: <mode>,<read_len>,<rest_len> <data>
    //                          b. If multi-client:     +CIPRXGET: <mode>,<cid>,<read_len>,<rest_len> <data>
    //          3. If<mode> = 4: 
    //                          a. If single-client:    +CIPRXGET: 4,<rest_len>
    //                          b. If multi-client:     +CIPRXGET: 4,<cid>,<rest_len>
    //          4. If an error occurs:                  +IP ERROR: <error message>
+RECEIVE: 
----------------------------------------
The ones relevant for Receiving HTTP (p.474):
----------------------------------------
+CHTTPS: RECV EVENT         - When there is data cached in the receiving buffer
+CHTTPSNOTIFY: PEER CLOSED  - The HTTPS session is closed by the server.

----------------------------------------
The ones relevant for Common Channel (p.474):
----------------------------------------
+CCHRECV: DATA, <session_id>,<len>

• We also need to add SSL/TLS support as the module does support that.
  Any ideas how to go about that?

[SRGDamia1]

The +IPCLOSE: and +CLIENT: URC's are only relevant if you're running as a TCP server instead of as a client. We're already listening for CIPRXGET and we're not managing HTTP or common channel.

[eabase]

Hi @SRGDamia1 and welcome back.

The +IPCLOSE: and +CLIENT: URC's are only relevant if you're running as a TCP server ...

That can't be true, because we're certainly not running this as a server, and still get those URC's on regular basis. It's seem that the FW of that module must be doing other things then. Either way, it is still useful and important (for debugging purposes) to know what was the reason for closing any type of connections.

[SRGDamia1]

Oops, sorry. CLIENT: is for when the module is server and a client disconnects; IPCLOSE: we're handling. You're getting the CLIENT URC?

[eabase]

I'm actually not sure now, since it's been 2 weeks since I was working on this. I was trying to download SW updates via web requests and also via FTP. I added the above comment because I occasionally got some weird closing behavior with those URC's. So I don't remember what exactly I was doing when these appeared. But certainly never running as a server of any kind.

[SRGDamia1]

Would you please check all the functionality now and close this if it's working?

[eabase]

@SRGDamia1
Hi, sorry for delay. I have since abandoned this module as the SimCom SIMxxx modules are so poorly documented, outdated and all have different AT sets, which is rather annoying. I did get the module to work pretty good (as described above) but I never did the full test. If I ever happen to get back to developing using the SIM modules, I'll keep you posted.
Thanks again for your effort and help. 💯

PS. I can't close someone else's issue.