Update Deepseek

src/common/chat/chat_bot.cpp

@@ -17,16 +17,14 @@ chat_bot::chat_bot(unsigned int MAX_L, unsigned int device_id){
     }
     this->last_prefill_time = {0, "us"};
     this->system_prompt = "";
+    this->token_history.reserve(MAX_L);
 }
 
 /// \brief Load the model
 /// \param model_path the path to the model
-/// \param MAX_L the max length
+/// \param model_info the model info
 /// \note The function will load the model
-/// \note The function will load the model weights
-/// \note The function will load the model tokenizer
-/// \note The function will load the model sampler
-void chat_bot::load_model(std::string model_path, unsigned int MAX_L){
+void chat_bot::load_model(std::string model_path, json model_info){
     // default models
     static const std::vector<std::pair<std::string, std::string>> default_models = {
         {"Llama-3.2-1B-q4nx", "1B"}, // 1B
@@ -46,7 +44,7 @@ void chat_bot::load_model(std::string model_path, unsigned int MAX_L){
         header_print("FLM", "Model already loaded: " << this->model_path);
         return;
     }
-
+    this->is_think_model = model_info["details"]["think"];
     this->model_path = model_path;
     for (const auto& model : default_models){
         if (model.second == model_path){
@@ -59,7 +57,7 @@ void chat_bot::load_model(std::string model_path, unsigned int MAX_L){
     this->lm_config->from_pretrained(this->model_path);
 
     this->npu = std::make_unique<npu_manager>(npu_device::device_npu2, device_id);
-    this->MAX_L = MAX_L;
+    this->MAX_L = model_info["default_context_length"];
     this->q4nx = std::make_unique<Q4NX>(this->model_path);
     if (this->lm_config->model_type == "llama"){
         this->lm_engine = std::make_unique<llama_npu>(*this->lm_config, this->npu.get(), this->MAX_L);
@@ -73,7 +71,8 @@ void chat_bot::load_model(std::string model_path, unsigned int MAX_L){
     this->q4nx.reset();
     this->is_model_loaded = true;
 
-
+    this->token_history.clear();
+    this->token_history.reserve(this->MAX_L);
     this->tokenizer = std::make_unique<Tokenizer>(this->model_path);
 
     this->lm_engine->clear_context();
@@ -100,7 +99,7 @@ void chat_bot::load_model(std::string model_path, unsigned int MAX_L){
     // insert the system prompt
     std::vector<int> system_tokens = this->tokenizer->encode(this->get_system_prompt(true, this->system_prompt));
     std::vector<int> system_prompts = this->apply_chat_template(system_tokens, SYSTEM, false);
-    this->insert(system_prompts);
+    this->insert(system_prompts, true);
 }
 
 /// \brief Set the sampler
@@ -140,7 +139,9 @@ void chat_bot::insert(std::vector<int>& tokens, bool is_system_prompt){
         header_print("warning", "Max length reached, stopping prefilling...");
         return;
     }
-
+    for (int token : tokens){
+        this->token_history.push_back(token);
+    }
     buffer<bf16> y;
     this->profiler_list[PREFILL_TIME].start();
     y = this->lm_engine->prefill(tokens);
@@ -180,10 +181,19 @@ std::string chat_bot::generate(std::ostream& os){
     std::string result;
     result.reserve(4096);
 
+    if (this->is_think_model){
+        std::string think_result = this->tokenizer->run_time_decoder(128013);
+        result += think_result;
+        os << think_result << std::flush;
+        think_result = this->tokenizer->run_time_decoder(198);
+        result += think_result;
+        os << think_result << std::flush;
+    }
+
     int last_sampled_token = this->last_token;
-    this->lm_engine->add_sampled_token(this->last_token);
+    this->token_history.push_back(this->last_token);
     this->profiler_list[TKOEN_DECODE_TIME].start();
-    if (this->tokenizer->is_normal_token(last_sampled_token) && last_sampled_token != -1){
+    if (this->tokenizer->is_normal_token(last_sampled_token, this->is_think_model) && last_sampled_token != -1){
         std::string token_str = this->tokenizer->run_time_decoder(last_sampled_token);
         result += token_str;
         os << token_str << std::flush;
@@ -211,17 +221,16 @@ std::string chat_bot::generate(std::ostream& os){
 
         this->profiler_list[TKOEN_DECODE_TIME].start();
         this->profiler_list[TKOEN_DECODE_TIME].stop(1);
-        if (this->tokenizer->is_normal_token(sampled_token)){ // filter out special tokens
+        if (this->tokenizer->is_normal_token(sampled_token, this->is_think_model)){ // filter out special tokens
             std::string token_str = this->tokenizer->run_time_decoder(sampled_token);
             os << token_str << std::flush;
             result += token_str;
         }
+        this->token_history.push_back(sampled_token);
         if (this->tokenizer->is_eos(sampled_token)){
+            this->lm_engine->forward(last_sampled_token);
             break;
         }
-        else{
-            this->lm_engine->add_sampled_token(sampled_token);
-        }
     }
     if (this->total_tokens >= this->MAX_L){
         header_print("warning", "Max length reached, stopping generation...");
@@ -250,6 +259,7 @@ std::string chat_bot::generate_with_prompt(std::vector<int>& tokens, std::ostrea
 void chat_bot::clear_context(){
     this->total_tokens = 0;
     this->last_token = -1;
+    this->token_history.clear();
     this->lm_engine->clear_context();
     this->total_tokens = 0;
     for (size_t i = 0; i < PROFILER_TYPE_NUM; i++){
@@ -273,6 +283,7 @@ std::string chat_bot::get_system_prompt(bool include_timestamp, const std::strin
 
     // Build system prompt with context information
     std::stringstream system_prompt;
+    system_prompt << "You are a helpful assistant.\n";
 
     // Add timestamp if requested
     if (include_timestamp) {
@@ -345,7 +356,7 @@ std::string chat_bot::show_profile(){
 /// \note The function will get the history
 /// \note The function will return the history
 std::pair<std::string, std::vector<int>> chat_bot::get_history(){
-    std::vector<int> history = this->lm_engine->get_history();
+    std::vector<int> history = this->token_history;
     std::string all_context = this->tokenizer->decode(history);
     return std::make_pair(all_context, history);
 }
@@ -354,7 +365,7 @@ std::pair<std::string, std::vector<int>> chat_bot::get_history(){
 /// \note The function will get the history string
 /// \note The function will return the history string
 std::string chat_bot::get_history_string(){
-    std::vector<int> history = this->lm_engine->get_history();
+    std::vector<int> history = this->token_history;
     std::string all_context = this->tokenizer->decode(history);
     return all_context;
 }
@@ -477,18 +488,34 @@ std::vector<int> chat_bot::apply_chat_template(std::vector<int>& tokens, role_ty
         new_tokens.push_back(this->tokenizer->begin_of_header_id());
         new_tokens.push_back(this->tokenizer->assistant_id());
         new_tokens.push_back(this->tokenizer->end_of_header_id());
-        new_tokens.push_back(271); // \n\n
+        if (this->is_think_model){
+            new_tokens.push_back(128013); // <think>
+            new_tokens.push_back(198); // </think>
+        }
+        else{
+            new_tokens.push_back(271); // \n\n
+        }
     }
     else if (role == SYSTEM){ // system prompt header is added in the hardware
-        // new_tokens.push_back(this->tokenizer->system_id());
+        new_tokens.push_back(this->tokenizer->begin_of_text_id());
+        new_tokens.push_back(this->tokenizer->begin_of_header_id());
+        new_tokens.push_back(this->tokenizer->system_id());
+        new_tokens.push_back(this->tokenizer->end_of_header_id());
+        new_tokens.push_back(271); // \n\n
     }
     new_tokens.insert(new_tokens.end(), tokens.begin(), tokens.end());
     new_tokens.push_back(128009); // eot
     if (role == USER && append_assistant_prefix){
         new_tokens.push_back(this->tokenizer->begin_of_header_id());
         new_tokens.push_back(this->tokenizer->assistant_id());
         new_tokens.push_back(this->tokenizer->end_of_header_id());
-        new_tokens.push_back(271); // \n\n
+        if (this->is_think_model){
+            new_tokens.push_back(128013); // <think>
+            new_tokens.push_back(198); // </think>
+        }
+        else{
+            new_tokens.push_back(271); // \n\n
+        }
     }
     return new_tokens;
 }

src/include/buffer.hpp

@@ -120,15 +120,14 @@ class bytes {
     /// \brief destructor
     virtual ~bytes() {
         if (is_owner_) {
-            data_ = nullptr;
             owned_data_.reset();
         }
+        data_ = nullptr;
 #ifdef __XRT__
         if (is_bo_owner_) {
-            bo_ = nullptr;
-            owned_bo_->~bo();
             owned_bo_.reset();
         }
+        bo_ = nullptr;
 #endif
     }
 
@@ -139,7 +138,6 @@ class bytes {
             if (is_owner_){
                 owned_data_.reset();
             }
-            owned_data_.reset();
             data_ = other.data_;
             size_ = other.size_;
             is_owner_ = false;
@@ -149,7 +147,6 @@ class bytes {
             }
             is_bo_owner_ = false;
             bo_ = other.bo_;
-            owned_bo_.reset();
 #endif
         }
         return *this;
@@ -168,7 +165,7 @@ class bytes {
             is_owner_ = other.is_owner_;
 #ifdef __XRT__
             if (is_bo_owner_){
-                owned_bo_->~bo();
+                owned_bo_.reset();
             }
             is_bo_owner_ = other.is_bo_owner_;
             owned_bo_ = std::move(other.owned_bo_);
@@ -233,17 +230,18 @@ class bytes {
 #ifdef __XRT__
         assert(!is_bo_owner_);
 #endif
-        owned_data_.reset();
+        if (is_owner_){
+            owned_data_.reset();
+        }
         data_ = nullptr;
         size_ = 0;
         is_owner_ = false;
 #ifdef __XRT__
         if (is_bo_owner_){
-            owned_bo_->~bo();
+            owned_bo_.reset();
         }
         is_bo_owner_ = false;
         bo_ = nullptr;
-        owned_bo_.reset();
 #endif
     }
 

src/include/causal_lm.hpp

@@ -42,14 +42,6 @@ class causal_lm {
     /// \brief clear the context
     virtual void clear_context() = 0;
 
-    /// \brief add the sampled token
-    /// \param token the token
-    virtual void add_sampled_token(int token) = 0;
-
-    /// \brief get the history
-    /// \return the history
-    virtual std::vector<int> get_history() = 0;
-
     /// \brief get the k cache
     /// \param layer_idx the layer index
     /// \param idx the index

src/include/chat/chat_bot.hpp

@@ -35,7 +35,8 @@ class chat_bot {
     bool is_model_loaded = false;
     std::string model_path = "";
     std::string current_model = "Llama-3.2-1B-Instruct";
-
+    bool is_think_model = false;
+    std::vector<int> token_history;
     std::unique_ptr<npu_manager> npu = nullptr;
 
     uint32_t MAX_L = 0;
@@ -90,7 +91,8 @@ class chat_bot {
 
     /// \brief Load the model
     /// \param model_path the model path
-    void load_model(std::string model_path, unsigned int MAX_L = 131072);
+    /// \param model_info the model info
+    void load_model(std::string model_path, json model_info);
 
     /// \brief Average the embeddings
     /// \param tokens the tokens

src/include/llama/llama_npu.hpp

@@ -44,14 +44,6 @@ class llama_npu : public causal_lm{
     /// \brief update the max length
     void clear_context() override;
 
-    /// \brief add the sampled token
-    /// \param token the token
-    void add_sampled_token(int token) override;
-
-    /// \brief get the history
-    /// \return the history
-    std::vector<int> get_history() override;
-
     /// \brief get the k cache
     /// \param layer_idx the layer index
     /// \param idx the index

src/include/tokenizer/tokenizer.hpp

@@ -61,8 +61,16 @@ class Tokenizer {
 
     /// \brief Check if the token is normal
     /// \param token the token
+    /// \param is_think_model the is think model
     /// \return true if the token is normal, false otherwise
-    inline bool is_normal_token(int token) {return ((token < 128000) || (token == 128013) || (token == 128014));}
+    inline bool is_normal_token(int token, bool is_think_model) {
+        if (is_think_model){
+            return ((token < 128000) || (token == 128013) || (token == 128014));
+        }
+        else{
+            return ((token < 128000));
+        }
+    }
 
     /// \brief Get the begin of text id
     /// \return the begin of text id
@@ -84,6 +92,10 @@ class Tokenizer {
     /// \return the assistant id
     inline int assistant_id() {return 78191;}
 
+    /// \brief Get the system id
+    /// \return the system id
+    inline int system_id() {return 9125;}
+
     /// \brief Get the end of text id
     /// \return the end of text id
     inline int end_of_text_id() {return 128001;}

src/include/utils/utils.hpp

@@ -12,6 +12,10 @@
 #include <chrono>
 #include <iostream>
 #include <fstream>
+#ifdef _WIN32
+#include <windows.h>
+#include <shlobj.h>
+#endif
 
 namespace time_utils {
 
@@ -298,4 +302,17 @@ inline bool check_file_exists(std::string name) {
     return file.good();
 }
 
+#ifdef _WIN32
+/// \brief get the user's Documents directory on Windows
+/// \return the user's Documents directory path
+inline std::string get_user_documents_directory() {
+    char buffer[MAX_PATH];
+    if (SUCCEEDED(SHGetFolderPathA(NULL, CSIDL_PERSONAL, NULL, 0, buffer))) {
+        return std::string(buffer);
+    }
+    // Fallback to current directory if Documents folder cannot be found
+    return ".";
+}
+#endif
+
 } // end of namespace utils

src/model_list.json

@@ -1,5 +1,5 @@
 {
-    "model_path": "../../models",
+    "model_path": "flm/models",
     "models": {
         "llama3.2": {
             "1B": {
@@ -46,6 +46,23 @@
                     "quantization_level": "AWQ-asym-uint4-g32"
                 }
             }
+        },
+        "deepseek": {
+            "8B": {
+                "name": "deepseek-distill-llama-8B-q4nx",
+                "url": "https://huggingface.co/FastFlowLM/Deepseek-R1-Distill-Llama-8B-NPU2",
+                "modified_at": "2025-05-30T00:00:00Z",
+                "size": 8000000000,
+                "default_context_length": 16384,
+                "details": {
+                    "format": "q4nx",
+                    "family": "llama",
+                    "think": true,
+                    "parameter_size": "8B",
+                    "quantization_level": "AWQ-asym-uint4-g32"
+                }
+            }
         }
     }
+
 }