[update] update llm_vlm encoder. update audio cache.

Author: LittleMouse

projects/llm_framework/main_asr/src/main.cpp

@@ -58,7 +58,7 @@ class llm_task {
     std::atomic_bool audio_flage_;
     std::atomic_bool awake_flage_;
     int awake_delay_       = 50;
-    int delay_audio_frame_ = 10;
+    int delay_audio_frame_ = 11;
     buffer_t *pcmdata;
 
     std::function<void(void)> pause;
@@ -187,18 +187,20 @@ class llm_task {
             count++;
             return;
         }
-        buffer_write_char(pcmdata, raw.data(), raw.length());
         buffer_position_set(pcmdata, 0);
-        count = 0;
+
         std::vector<float> floatSamples;
         {
             int16_t audio_val;
-            while (buffer_read_u16(pcmdata, (unsigned short *)&audio_val, 1)) {
-                float normalizedSample = (float)audio_val / INT16_MAX;
+            while (buffer_read_i16(pcmdata, &audio_val, 1)) {
+                float normalizedSample = static_cast<float>(audio_val) / INT16_MAX;
                 floatSamples.push_back(normalizedSample);
             }
         }
-        buffer_position_set(pcmdata, 0);
+
+        buffer_resize(pcmdata, 0);
+        count = 0;
+
         if (awake_flage_ && recognizer_stream_) {
             recognizer_stream_.reset();
             awake_flage_ = false;

projects/llm_framework/main_kws/src/main.cpp

@@ -59,7 +59,7 @@ class llm_task {
     bool enwake_audio_;
     std::atomic_bool audio_flage_;
     task_callback_t out_callback_;
-    int delay_audio_frame_ = 10;
+    int delay_audio_frame_ = 11;
     buffer_t *pcmdata;
     std::string wake_wav_file_;
 
@@ -229,22 +229,24 @@ class llm_task {
     {
         static int count = 0;
         if (count < delay_audio_frame_) {
-            buffer_write_char(pcmdata, raw.c_str(), raw.length());
+            buffer_write_char(pcmdata, raw.data(), raw.length());
             count++;
             return;
         }
-        buffer_write_char(pcmdata, raw.data(), raw.length());
         buffer_position_set(pcmdata, 0);
-        count = 0;
+
         std::vector<float> floatSamples;
         {
             int16_t audio_val;
-            while (buffer_read_u16(pcmdata, (unsigned short *)&audio_val, 1)) {
-                float normalizedSample = (float)audio_val / INT16_MAX;
+            while (buffer_read_i16(pcmdata, &audio_val, 1)) {
+                float normalizedSample = static_cast<float>(audio_val) / INT16_MAX;
                 floatSamples.push_back(normalizedSample);
             }
         }
-        buffer_position_set(pcmdata, 0);
+
+        buffer_resize(pcmdata, 0);
+        count = 0;
+
         spotter_stream_->AcceptWaveform(mode_config_.feat_config.sampling_rate, floatSamples.data(),
                                         floatSamples.size());
         while (spotter_->IsReady(spotter_stream_.get())) {

projects/llm_framework/main_llm/src/main.cpp

@@ -130,6 +130,7 @@ class llm_task {
             std::string base_model = base_model_path_ + model_ + "/";
             SLOGI("base_model %s", base_model.c_str());
 
+            CONFIG_AUTO_SET(file_body["mode_param"], system_prompt);   
             CONFIG_AUTO_SET(file_body["mode_param"], tokenizer_type);
             CONFIG_AUTO_SET(file_body["mode_param"], filename_tokenizer_model);
             CONFIG_AUTO_SET(file_body["mode_param"], url_tokenizer_model);

projects/llm_framework/main_vad/src/main.cpp

@@ -60,7 +60,7 @@ class llm_task {
     std::string superior_id_;
     task_callback_t out_callback_;
     int awake_delay_       = 50;
-    int delay_audio_frame_ = 3;
+    int delay_audio_frame_ = 4;
     buffer_t *pcmdata;
     std::string wake_wav_file_;
 
@@ -158,18 +158,19 @@ class llm_task {
             count++;
             return;
         }
-        buffer_write_char(pcmdata, raw.data(), raw.length());
         buffer_position_set(pcmdata, 0);
-        count = 0;
+
         std::vector<float> floatSamples;
         {
             int16_t audio_val;
-            while (buffer_read_u16(pcmdata, (unsigned short *)&audio_val, 1)) {
-                float normalizedSample = (float)audio_val / INT16_MAX;
+            while (buffer_read_i16(pcmdata, &audio_val, 1)) {
+                float normalizedSample = static_cast<float>(audio_val) / INT16_MAX;
                 floatSamples.push_back(normalizedSample);
             }
         }
-        buffer_position_set(pcmdata, 0);
+        buffer_resize(pcmdata, 0);
+        count = 0;
+
         vad_->AcceptWaveform(floatSamples.data(), floatSamples.size());
 
         if (vad_->IsSpeechDetected() && !printed) {