TinyML基本信息了解

      TinyML开发板是采用超低功耗 Syntiant ® NDP101神经决策处理器™为核心，可以实现低功耗下的语音识别和传感器数据分析功能的一个小型独立功能板。板子设计的比较小巧，整板尺寸仅为24mmX28mm。在使用过程中发现这款板子不仅设计精致而且使用上也比较方便，在整个开发周期内，仅仅需要一个USB端口连接到板子上既可实现对语音数据收集、模型下载验证，整板测试的功能。开发过程中遇到的坑大部分都是软件上的一些问题，后面会把遇到的坑梳理一下。

      在开发的整个过程中主要是依据官方的一些文档以及edgeimpulse平台的一些教程逐步开展的，使用到文档和教程链接详见文末。

板子主要特点：

• 神经决策处理器使用NDP101，连接了两个传感器（BMI160 6轴运动传感器SPH0641LM4H麦克风）
• 主机处理器：SAMD21 Cortex-M0+ 32位低功耗48MHz ARM MCU（内置256KB FLASH和32KB SRAM，5个数字I/O，与Arduino MKR系列板兼容，其中包含1路UART接口和1路I2C接口）
• 2MB板载串行闪存
• 一个用户定义的RGB LED
• uSD卡插槽
• 电路板可使用5V micro-USB或3.7V LiPo电池供电

      TinyML的结构框图如上图所示，可以看出板子主要由SMAD11 MCU芯片和NDP101神经决策处理器芯片以及一些传感器，麦克风等外围器件构成。通过结构框图大概可以了解到整个系统是由SMAD11-NDP101组成的主从结构系统，通过SPI进行通讯。NDP101扮演着协处理器的作用，应该是主要用来处理MCU并不擅长的神经网络运算的工作。

基于edgeimpulse平台训练语音识别神经网络

1 语音数据收集

数据及主要包括中文语音“打开”“关闭”“跑马灯”和环境噪声四部分组成。

配置“audio”以及“Neural Network”的项目参数，然后重新生成和训练识别网络。

调整参数，训练网络

生成工程代码

firmware-syntiant-tinyml项目开发Arduino项目

      克隆GitHub固件代码firmware-syntiant-tinyml，修改代码后，将Arduino输出的.bin文件并将其重命名为 firmware.ino.bin然后使用之前相同的脚本来更新功能的程序固件。

在Windows 中，.bin文件将位于输出目录中:(C：\Users\username\AppData\Local\Temp\arduino_build_106482).

修改源码

1 修改isrTimer4函数取消默认固件的识别到关键词后的默认闪灯（默认固件是识别到关机词就会操作LED）

// Timer 4 interrupt. Handles ALL touches of NDP. Also services USB Audio
void isrTimer4(struct tc_module *const module_inst)
{
    digitalWrite(0, LOW);
    int s;
    unsigned int len;

    SCB->SCR &= !SCB_SCR_SLEEPDEEP_Msk; // Don't Allow Deep Sleep
    if ((ledTimerCount < 0xffff) && (ledTimerCount > 0))
    {
        ledTimerCount--;
        if (ledTimerCount == 0)
        {
            timer4TimedOut = 1;
        }
    }


#ifdef WITH_AUDIO
    if (runningFromFlash) {

        uint32_t tankRead;
        int i;

        for (i = 0; i < 32; i += 4)
        {
            tankRead =
                indirectRead(tankAddress + ((currentPointer - 32 + i) % tankSize));
            audioBuf[i / 2] = tankRead & 0xffff;
            audioBuf[(i / 2) + 1] = (tankRead >> 16) & 0xffff;
        }
        currentPointer += 32;
        currentPointer %= tankSize;
    }
    AudioUSB.write(audioBuf, 32); // write samples to AudioUSB
#else

    if(runningFromFlash) {
        currentPointer = indirectRead(startingFWAddress);

        int32_t diffPointer = ((int32_t)currentPointer - prevPointer);

        if(diffPointer < 0) {
            diffPointer = (64000 - prevPointer) + currentPointer;
        }
        if(diffPointer >= dataLengthToBeSaved) {
            len = sizeof(dataBuf);
            int ret = NDP.extractData((uint8_t *)dataBuf, &len);

            if(ret != SYNTIANT_NDP_ERROR_NONE) {
                ei_printf("Extracting data failed with error : %d\r\n", ret);
            }
            else {
                if(imu_active == false) {
                    imu_active = true;
                    for(int i = 0; i < (dataLengthToBeSaved / 2); i++) {
                        imu[i] = dataBuf[i];
                    }

                    imu_active = false;
                }
            }

            prevPointer = currentPointer;
        }
    }
#endif

    if (doInt)
    {
        doInt = 0;
        // Poll NDP for cause of interrupt (if running from flash)
        if (runningFromFlash)
        {
            match = NDP.poll();

            if (match)
            {
                // Light Arduino LED
                // digitalWrite(LED_BUILTIN, HIGH);

                ei_classification_output(match -1);

                printBattery(); // Print current battery level

            }
        }
        else
        {
            match = 1;
        }
    }
    led_timer_count(300);
    digitalWrite(0, HIGH);
}

2 修改syntiant_loop函数的关灯操作，将识别超时后的关灯操作去掉，使命令词识别后可以长时间执行点亮状态。

void syntiant_loop(void)
{
    // Loop to stay in Standby Mode unless we get a ": " from USB
    // OR interrupt from NDP
    while (1)
    {
        if (match)
        {
            processMatch();
        }
        if (timer4TimedOut)
        {
            timer4TimedOut = 0;
            // Turn Off Arduino LED
            // digitalWrite(LED_RED, LOW);
            // digitalWrite(LED_BLUE, LOW);
            // digitalWrite(LED_GREEN, LOW);
        }
        // check USB serial port for ": " command from host
        // if (Serial.read() == ':')
        // {
        //     break;
        // }
        if(ei_command_line_handle() == true) {
            break;
        }

        // Deep sleep only if USB disconnected.
        SCB->SCR &= !SCB_SCR_SLEEPDEEP_Msk; // remove deep sleep bit

        if ((USB->DEVICE.STATUS.reg & 0xc0) != idle)
        {
            // we are connected to USB
            USBConnected = 0;
            timer4.enableInterrupt(true); // enable interrupt for Audio
            if (detached == 1)
            {
                detached = 0; // assume attached to host USB
                Serial2.println("Recommected to USB");
            }
        }
        else
            USBConnected += 1;

        if (USBConnected > 0xfff00)
        {
            USBConnected = 0;

            // Only deep sleep (Standby) if LED timer has expired.
            // See if LED timer = 0, & timed out flag not set
            if ((!ledTimerCount) && (!timer4TimedOut))
            {
                SCB->SCR |= SCB_SCR_SLEEPDEEP_Msk; // enable Deep Sleep

                // This saves 0.63mA when battery powered. But then needs
                // keyword interrupt to wake CPU up
                USBDevice.detach();
                detached = 1;

                // stop timer as we are going to deep sleep
                timer4.enableInterrupt(false);

                // Put Flash into Deep Power Down
                digitalWrite(FLASH_CS, LOW);
                SPI1.transfer(FLASH_DP); // enable RESET
                digitalWrite(FLASH_CS, HIGH);
                delayMicroseconds(1);
                delay(1);

                delay(1000);
                Serial2.println("Deep Sleep");
                delay(2000);
            }
            else
            {
                timer4.enableInterrupt(true);
            }
        }

        digitalWrite(1, HIGH);

        __DSB(); // Data sync to ensure outgoing memory accesses complete
        __WFI();
        if (REG_SYSCTRL_DFLLMUL != SAVE_REG_SYSCTRL_DFLLMUL)
        {
            REG_SYSCTRL_DFLLMUL = SYSCTRL_DFLLMUL_MUL(0xBB80) | SYSCTRL_DFLLMUL_FSTEP(1) | SYSCTRL_DFLLMUL_CSTEP(1);
        }

        digitalWrite(1, LOW);

        // Woken up. See if attached to USB host
        if (detached)
        {
            USBDevice.attach();
            Serial.begin(115200);
            detached = 0;
        }
    }
    // Stop timer4 as we will access NDP from main()
    timer4.enableInterrupt(false); // disable 1mS timer interrupt

    // ':' received -- perform a management command
    runManagementCommand();

    timer4.enableInterrupt(true); // enable 1mS timer interrupt
}

3 基于Timer4定时器增加跑马灯逻辑，跑马灯逻辑是基于识别定时器中断实现的。识别定时器的周期1ms，跑马灯切换频率是以1ms时延为计数基准，设定为300个计数周期切换一次也就是300ms周期。

uint32_t walk_led_timer_count = 0;
uint32_t walk_led_index = 0;
uint32_t walk_led_enable = 0;
void led_timer_count(uint32_t delay_ms){
    
    walk_led_timer_count++;
    if(walk_led_timer_count >delay_ms){
        walk_led_timer_count = 0;

        if(walk_led_enable){
            
            if(walk_led_index==0){
                digitalWrite(LED_RED, LOW);
                digitalWrite(LED_BLUE, LOW);
                digitalWrite(LED_GREEN, LOW);
                digitalWrite(LED_RED, HIGH);
            }
            if (walk_led_index==1)
            {
                digitalWrite(LED_RED, LOW);
                digitalWrite(LED_BLUE, LOW);
                digitalWrite(LED_GREEN, LOW);
                digitalWrite(LED_GREEN, HIGH);
            }
            if (walk_led_index==2)
            {
                digitalWrite(LED_RED, LOW);
                digitalWrite(LED_BLUE, LOW);
                digitalWrite(LED_GREEN, LOW);
                digitalWrite(LED_BLUE, HIGH);
            }
            
            
            walk_led_index++;
            if (walk_led_index>2)
            {
               walk_led_index = 0;
            }
            
        }
    }
}

在Timer4中断服务函数中进行调用，延时设置为300ms。

// Timer 4 interrupt. Handles ALL touches of NDP. Also services USB Audio
void isrTimer4(struct tc_module *const module_inst)
{
    ....
    led_timer_count(300);
    digitalWrite(0, HIGH);
}

最后调整识别回调函数on_classification_changed。当时别到“打开”关键词时会打开绿灯，当时别到“关闭”关键词时会打开红灯，当时别到“跑马灯”关键词时会循环依次打开红绿蓝灯的跑马灯效果。

extern uint32_t walk_led_enable;
/**
 * @brief      Called when a inference matches 1 of the features
 *
 * @param[in]  event          The event
 * @param[in]  confidence     The confidence
 * @param[in]  anomaly_score  The anomaly score
 */
void on_classification_changed(const char *event, float confidence, float anomaly_score) {
    walk_led_enable = 0;
    // here you can write application code, e.g. to toggle LEDs based on keywords
    if (strcmp(event, "on") == 0) {
        // Toggle LED
        digitalWrite(LED_RED, LOW);
        digitalWrite(LED_BLUE, LOW);
        digitalWrite(LED_GREEN, LOW);
        
        digitalWrite(LED_GREEN, HIGH);
        Serial.println("turn on");
    }

    if (strcmp(event, "off") == 0) {
        // Toggle LED
        digitalWrite(LED_RED, LOW);
        digitalWrite(LED_BLUE, LOW);
        digitalWrite(LED_GREEN, LOW);
        
        digitalWrite(LED_RED, HIGH);
        Serial.println("turn off");
        
    }
    if (strcmp(event, "walk") == 0) {
        // Toggle LED
        digitalWrite(LED_RED, LOW);
        digitalWrite(LED_BLUE, LOW);
        digitalWrite(LED_GREEN, LOW);
        
//        digitalWrite(LED_BLUE, HIGH);
        walk_led_enable = 1;
        Serial.println("turn walk");
    }
}

参考链接：

1. Syntiant Tiny ML Board - Edge Impulse Documentation
2. Responding to your voice - Syntiant - RC Commands - Edge Impulse Documentation
3. On your Syntiant TinyML Board - Edge Impulse Documentation

遇到的问题：

1 写网络模型后报错

解决办法：

串口助手发送“:F”字符串

2 移植固件需要拷贝model_variables.h文件，默认该文件置顶的ei_model_types.h路径不正确。