diff --git a/src/common/chat/chat_bot.cpp b/src/common/chat/chat_bot.cpp index afe998675..eb2ae47f7 100644 --- a/src/common/chat/chat_bot.cpp +++ b/src/common/chat/chat_bot.cpp @@ -17,16 +17,14 @@ chat_bot::chat_bot(unsigned int MAX_L, unsigned int device_id){ } this->last_prefill_time = {0, "us"}; this->system_prompt = ""; + this->token_history.reserve(MAX_L); } /// \brief Load the model /// \param model_path the path to the model -/// \param MAX_L the max length +/// \param model_info the model info /// \note The function will load the model -/// \note The function will load the model weights -/// \note The function will load the model tokenizer -/// \note The function will load the model sampler -void chat_bot::load_model(std::string model_path, unsigned int MAX_L){ +void chat_bot::load_model(std::string model_path, json model_info){ // default models static const std::vector> default_models = { {"Llama-3.2-1B-q4nx", "1B"}, // 1B @@ -46,7 +44,7 @@ void chat_bot::load_model(std::string model_path, unsigned int MAX_L){ header_print("FLM", "Model already loaded: " << this->model_path); return; } - + this->is_think_model = model_info["details"]["think"]; this->model_path = model_path; for (const auto& model : default_models){ if (model.second == model_path){ @@ -59,7 +57,7 @@ void chat_bot::load_model(std::string model_path, unsigned int MAX_L){ this->lm_config->from_pretrained(this->model_path); this->npu = std::make_unique(npu_device::device_npu2, device_id); - this->MAX_L = MAX_L; + this->MAX_L = model_info["default_context_length"]; this->q4nx = std::make_unique(this->model_path); if (this->lm_config->model_type == "llama"){ this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); @@ -73,7 +71,8 @@ void chat_bot::load_model(std::string model_path, unsigned int MAX_L){ this->q4nx.reset(); this->is_model_loaded = true; - + this->token_history.clear(); + this->token_history.reserve(this->MAX_L); this->tokenizer = std::make_unique(this->model_path); this->lm_engine->clear_context(); @@ -100,7 +99,7 @@ void chat_bot::load_model(std::string model_path, unsigned int MAX_L){ // insert the system prompt std::vector system_tokens = this->tokenizer->encode(this->get_system_prompt(true, this->system_prompt)); std::vector system_prompts = this->apply_chat_template(system_tokens, SYSTEM, false); - this->insert(system_prompts); + this->insert(system_prompts, true); } /// \brief Set the sampler @@ -140,7 +139,9 @@ void chat_bot::insert(std::vector& tokens, bool is_system_prompt){ header_print("warning", "Max length reached, stopping prefilling..."); return; } - + for (int token : tokens){ + this->token_history.push_back(token); + } buffer y; this->profiler_list[PREFILL_TIME].start(); y = this->lm_engine->prefill(tokens); @@ -180,10 +181,19 @@ std::string chat_bot::generate(std::ostream& os){ std::string result; result.reserve(4096); + if (this->is_think_model){ + std::string think_result = this->tokenizer->run_time_decoder(128013); + result += think_result; + os << think_result << std::flush; + think_result = this->tokenizer->run_time_decoder(198); + result += think_result; + os << think_result << std::flush; + } + int last_sampled_token = this->last_token; - this->lm_engine->add_sampled_token(this->last_token); + this->token_history.push_back(this->last_token); this->profiler_list[TKOEN_DECODE_TIME].start(); - if (this->tokenizer->is_normal_token(last_sampled_token) && last_sampled_token != -1){ + if (this->tokenizer->is_normal_token(last_sampled_token, this->is_think_model) && last_sampled_token != -1){ std::string token_str = this->tokenizer->run_time_decoder(last_sampled_token); result += token_str; os << token_str << std::flush; @@ -211,17 +221,16 @@ std::string chat_bot::generate(std::ostream& os){ this->profiler_list[TKOEN_DECODE_TIME].start(); this->profiler_list[TKOEN_DECODE_TIME].stop(1); - if (this->tokenizer->is_normal_token(sampled_token)){ // filter out special tokens + if (this->tokenizer->is_normal_token(sampled_token, this->is_think_model)){ // filter out special tokens std::string token_str = this->tokenizer->run_time_decoder(sampled_token); os << token_str << std::flush; result += token_str; } + this->token_history.push_back(sampled_token); if (this->tokenizer->is_eos(sampled_token)){ + this->lm_engine->forward(last_sampled_token); break; } - else{ - this->lm_engine->add_sampled_token(sampled_token); - } } if (this->total_tokens >= this->MAX_L){ header_print("warning", "Max length reached, stopping generation..."); @@ -250,6 +259,7 @@ std::string chat_bot::generate_with_prompt(std::vector& tokens, std::ostrea void chat_bot::clear_context(){ this->total_tokens = 0; this->last_token = -1; + this->token_history.clear(); this->lm_engine->clear_context(); this->total_tokens = 0; for (size_t i = 0; i < PROFILER_TYPE_NUM; i++){ @@ -273,6 +283,7 @@ std::string chat_bot::get_system_prompt(bool include_timestamp, const std::strin // Build system prompt with context information std::stringstream system_prompt; + system_prompt << "You are a helpful assistant.\n"; // Add timestamp if requested if (include_timestamp) { @@ -345,7 +356,7 @@ std::string chat_bot::show_profile(){ /// \note The function will get the history /// \note The function will return the history std::pair> chat_bot::get_history(){ - std::vector history = this->lm_engine->get_history(); + std::vector history = this->token_history; std::string all_context = this->tokenizer->decode(history); return std::make_pair(all_context, history); } @@ -354,7 +365,7 @@ std::pair> chat_bot::get_history(){ /// \note The function will get the history string /// \note The function will return the history string std::string chat_bot::get_history_string(){ - std::vector history = this->lm_engine->get_history(); + std::vector history = this->token_history; std::string all_context = this->tokenizer->decode(history); return all_context; } @@ -477,10 +488,20 @@ std::vector chat_bot::apply_chat_template(std::vector& tokens, role_ty new_tokens.push_back(this->tokenizer->begin_of_header_id()); new_tokens.push_back(this->tokenizer->assistant_id()); new_tokens.push_back(this->tokenizer->end_of_header_id()); - new_tokens.push_back(271); // \n\n + if (this->is_think_model){ + new_tokens.push_back(128013); // + new_tokens.push_back(198); // + } + else{ + new_tokens.push_back(271); // \n\n + } } else if (role == SYSTEM){ // system prompt header is added in the hardware - // new_tokens.push_back(this->tokenizer->system_id()); + new_tokens.push_back(this->tokenizer->begin_of_text_id()); + new_tokens.push_back(this->tokenizer->begin_of_header_id()); + new_tokens.push_back(this->tokenizer->system_id()); + new_tokens.push_back(this->tokenizer->end_of_header_id()); + new_tokens.push_back(271); // \n\n } new_tokens.insert(new_tokens.end(), tokens.begin(), tokens.end()); new_tokens.push_back(128009); // eot @@ -488,7 +509,13 @@ std::vector chat_bot::apply_chat_template(std::vector& tokens, role_ty new_tokens.push_back(this->tokenizer->begin_of_header_id()); new_tokens.push_back(this->tokenizer->assistant_id()); new_tokens.push_back(this->tokenizer->end_of_header_id()); - new_tokens.push_back(271); // \n\n + if (this->is_think_model){ + new_tokens.push_back(128013); // + new_tokens.push_back(198); // + } + else{ + new_tokens.push_back(271); // \n\n + } } return new_tokens; } diff --git a/src/include/buffer.hpp b/src/include/buffer.hpp index 553d27bff..e4679338f 100644 --- a/src/include/buffer.hpp +++ b/src/include/buffer.hpp @@ -120,15 +120,14 @@ class bytes { /// \brief destructor virtual ~bytes() { if (is_owner_) { - data_ = nullptr; owned_data_.reset(); } + data_ = nullptr; #ifdef __XRT__ if (is_bo_owner_) { - bo_ = nullptr; - owned_bo_->~bo(); owned_bo_.reset(); } + bo_ = nullptr; #endif } @@ -139,7 +138,6 @@ class bytes { if (is_owner_){ owned_data_.reset(); } - owned_data_.reset(); data_ = other.data_; size_ = other.size_; is_owner_ = false; @@ -149,7 +147,6 @@ class bytes { } is_bo_owner_ = false; bo_ = other.bo_; - owned_bo_.reset(); #endif } return *this; @@ -168,7 +165,7 @@ class bytes { is_owner_ = other.is_owner_; #ifdef __XRT__ if (is_bo_owner_){ - owned_bo_->~bo(); + owned_bo_.reset(); } is_bo_owner_ = other.is_bo_owner_; owned_bo_ = std::move(other.owned_bo_); @@ -233,17 +230,18 @@ class bytes { #ifdef __XRT__ assert(!is_bo_owner_); #endif - owned_data_.reset(); + if (is_owner_){ + owned_data_.reset(); + } data_ = nullptr; size_ = 0; is_owner_ = false; #ifdef __XRT__ if (is_bo_owner_){ - owned_bo_->~bo(); + owned_bo_.reset(); } is_bo_owner_ = false; bo_ = nullptr; - owned_bo_.reset(); #endif } diff --git a/src/include/causal_lm.hpp b/src/include/causal_lm.hpp index 6419e3330..bb89fd6f2 100644 --- a/src/include/causal_lm.hpp +++ b/src/include/causal_lm.hpp @@ -42,14 +42,6 @@ class causal_lm { /// \brief clear the context virtual void clear_context() = 0; - /// \brief add the sampled token - /// \param token the token - virtual void add_sampled_token(int token) = 0; - - /// \brief get the history - /// \return the history - virtual std::vector get_history() = 0; - /// \brief get the k cache /// \param layer_idx the layer index /// \param idx the index diff --git a/src/include/chat/chat_bot.hpp b/src/include/chat/chat_bot.hpp index 0342bce02..80d73e09f 100644 --- a/src/include/chat/chat_bot.hpp +++ b/src/include/chat/chat_bot.hpp @@ -35,7 +35,8 @@ class chat_bot { bool is_model_loaded = false; std::string model_path = ""; std::string current_model = "Llama-3.2-1B-Instruct"; - + bool is_think_model = false; + std::vector token_history; std::unique_ptr npu = nullptr; uint32_t MAX_L = 0; @@ -90,7 +91,8 @@ class chat_bot { /// \brief Load the model /// \param model_path the model path - void load_model(std::string model_path, unsigned int MAX_L = 131072); + /// \param model_info the model info + void load_model(std::string model_path, json model_info); /// \brief Average the embeddings /// \param tokens the tokens diff --git a/src/include/llama/llama_npu.hpp b/src/include/llama/llama_npu.hpp index adfb802b3..cbd1bff88 100644 --- a/src/include/llama/llama_npu.hpp +++ b/src/include/llama/llama_npu.hpp @@ -44,14 +44,6 @@ class llama_npu : public causal_lm{ /// \brief update the max length void clear_context() override; - /// \brief add the sampled token - /// \param token the token - void add_sampled_token(int token) override; - - /// \brief get the history - /// \return the history - std::vector get_history() override; - /// \brief get the k cache /// \param layer_idx the layer index /// \param idx the index diff --git a/src/include/tokenizer/tokenizer.hpp b/src/include/tokenizer/tokenizer.hpp index e7b44adfc..b382932ea 100644 --- a/src/include/tokenizer/tokenizer.hpp +++ b/src/include/tokenizer/tokenizer.hpp @@ -61,8 +61,16 @@ class Tokenizer { /// \brief Check if the token is normal /// \param token the token + /// \param is_think_model the is think model /// \return true if the token is normal, false otherwise - inline bool is_normal_token(int token) {return ((token < 128000) || (token == 128013) || (token == 128014));} + inline bool is_normal_token(int token, bool is_think_model) { + if (is_think_model){ + return ((token < 128000) || (token == 128013) || (token == 128014)); + } + else{ + return ((token < 128000)); + } + } /// \brief Get the begin of text id /// \return the begin of text id @@ -84,6 +92,10 @@ class Tokenizer { /// \return the assistant id inline int assistant_id() {return 78191;} + /// \brief Get the system id + /// \return the system id + inline int system_id() {return 9125;} + /// \brief Get the end of text id /// \return the end of text id inline int end_of_text_id() {return 128001;} diff --git a/src/include/utils/utils.hpp b/src/include/utils/utils.hpp index b1e696ece..583c4643d 100644 --- a/src/include/utils/utils.hpp +++ b/src/include/utils/utils.hpp @@ -12,6 +12,10 @@ #include #include #include +#ifdef _WIN32 +#include +#include +#endif namespace time_utils { @@ -298,4 +302,17 @@ inline bool check_file_exists(std::string name) { return file.good(); } +#ifdef _WIN32 +/// \brief get the user's Documents directory on Windows +/// \return the user's Documents directory path +inline std::string get_user_documents_directory() { + char buffer[MAX_PATH]; + if (SUCCEEDED(SHGetFolderPathA(NULL, CSIDL_PERSONAL, NULL, 0, buffer))) { + return std::string(buffer); + } + // Fallback to current directory if Documents folder cannot be found + return "."; +} +#endif + } // end of namespace utils \ No newline at end of file diff --git a/src/lib/libllama_npu.so b/src/lib/libllama_npu.so deleted file mode 100644 index aa9cc9310..000000000 Binary files a/src/lib/libllama_npu.so and /dev/null differ diff --git a/src/lib/libq4_npu_eXpress.so b/src/lib/libq4_npu_eXpress.so deleted file mode 100644 index 036557f9e..000000000 Binary files a/src/lib/libq4_npu_eXpress.so and /dev/null differ diff --git a/src/lib/llama_npu.dll b/src/lib/llama_npu.dll index fa9b7d8a1..fecd6d1d2 100644 Binary files a/src/lib/llama_npu.dll and b/src/lib/llama_npu.dll differ diff --git a/src/lib/llama_npu.exp b/src/lib/llama_npu.exp index 7dc9bdc05..e6937135a 100644 Binary files a/src/lib/llama_npu.exp and b/src/lib/llama_npu.exp differ diff --git a/src/lib/llama_npu.lib b/src/lib/llama_npu.lib index aa1367061..56057896c 100644 Binary files a/src/lib/llama_npu.lib and b/src/lib/llama_npu.lib differ diff --git a/src/lib/q4_npu_eXpress.dll b/src/lib/q4_npu_eXpress.dll index f21293013..0c3e75f73 100644 Binary files a/src/lib/q4_npu_eXpress.dll and b/src/lib/q4_npu_eXpress.dll differ diff --git a/src/lib/test_llama_npu.exp b/src/lib/test_llama_npu.exp deleted file mode 100644 index d5bb324ae..000000000 Binary files a/src/lib/test_llama_npu.exp and /dev/null differ diff --git a/src/lib/test_llama_npu.lib b/src/lib/test_llama_npu.lib deleted file mode 100644 index 7d19bfee1..000000000 Binary files a/src/lib/test_llama_npu.lib and /dev/null differ diff --git a/src/model_list.json b/src/model_list.json index 901272d01..cce5b8091 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -1,5 +1,5 @@ { - "model_path": "../../models", + "model_path": "flm/models", "models": { "llama3.2": { "1B": { @@ -46,6 +46,23 @@ "quantization_level": "AWQ-asym-uint4-g32" } } + }, + "deepseek": { + "8B": { + "name": "deepseek-distill-llama-8B-q4nx", + "url": "https://huggingface.co/FastFlowLM/Deepseek-R1-Distill-Llama-8B-NPU2", + "modified_at": "2025-05-30T00:00:00Z", + "size": 8000000000, + "default_context_length": 16384, + "details": { + "format": "q4nx", + "family": "llama", + "think": true, + "parameter_size": "8B", + "quantization_level": "AWQ-asym-uint4-g32" + } + } } } + } diff --git a/src/pull/download_model.cpp b/src/pull/download_model.cpp index 0ac290984..40f7c4aa2 100644 --- a/src/pull/download_model.cpp +++ b/src/pull/download_model.cpp @@ -12,6 +12,16 @@ namespace download_utils { +/// \brief Hide the cursor +void hide_cursor() { + std::cout << "\033[?25l" << std::flush; +} + +/// \brief Show the cursor +void show_cursor() { + std::cout << "\033[?25h" << std::flush; +} + /// \brief Callback function for libcurl to write data to a file /// \param ptr the pointer to the data /// \param size the size of the data @@ -75,6 +85,9 @@ bool download_file(const std::string& url, const std::string& local_path, return false; } + // Hide cursor before starting download + hide_cursor(); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data_to_file); curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); @@ -95,6 +108,9 @@ bool download_file(const std::string& url, const std::string& local_path, fclose(fp); curl_easy_cleanup(curl); + // Show cursor after download completes + show_cursor(); + if (res != CURLE_OK) { std::cerr << "CURL error: " << curl_easy_strerror(res) << std::endl; std::filesystem::remove(local_path); // Remove partial download @@ -146,6 +162,9 @@ bool download_multiple_files(const std::vector #include #include +#include +#include +#include +#include /// \brief Command map for command line input std::map cmd_map = { @@ -40,7 +44,7 @@ Runner::Runner(model_list& supported_models, ModelDownloader& downloader, std::s this->downloader.pull_model(this->tag); } nlohmann::json model_info = this->supported_models.get_model_info(this->tag); - this->chat_engine->load_model(this->supported_models.get_model_path(this->tag), model_info["default_context_length"]); + this->chat_engine->load_model(this->supported_models.get_model_path(this->tag), model_info); // Set console to UTF-8 mode for proper Unicode handling SetConsoleOutputCP(CP_UTF8); @@ -336,17 +340,26 @@ void Runner::cmd_load(std::vector& input_list) { std::string model_name = input_list[1]; this->tag = model_name; if (!this->downloader.is_model_downloaded(this->tag)) { - this->downloader.model_not_found(this->tag); - exit(1); + this->downloader.pull_model(this->tag); } nlohmann::json model_info = this->supported_models.get_model_info(this->tag); - this->chat_engine->load_model(this->supported_models.get_model_path(this->tag), model_info["default_context_length"]); + this->chat_engine->load_model(this->supported_models.get_model_path(this->tag), model_info); } /// \brief Save the history /// \param input_list, std::vector void Runner::cmd_save(std::vector& input_list) { std::pair> history = this->chat_engine->get_history(); + + // Get the Documents directory and create the history directory + std::string documents_dir = utils::get_user_documents_directory(); + std::string history_dir = documents_dir + "/flm/history"; + + // Create the history directory if it doesn't exist + if (!std::filesystem::exists(history_dir)) { + std::filesystem::create_directories(history_dir); + } + // save file to history_hh_mm_mm_dd_yyyy.txt // 1) get current date std::time_t t = std::time(nullptr); @@ -364,8 +377,8 @@ void Runner::cmd_save(std::vector& input_list) { << (tm.tm_year + 1900); std::string date_str = date_ss.str(); - // 2) build filename - std::string file_name = "history_" + date_str + ".txt"; + // 2) build filename in the history directory + std::string file_name = history_dir + "/history_" + date_str + ".txt"; std::ofstream file(file_name); if (file.is_open()) { file << history.first << std::endl; diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 94fd56f91..7de29d454 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -41,7 +41,7 @@ void RestHandler::ensure_model_loaded(const std::string& model_tag) { } std::string tag_copy = model_tag; // Create non-const copy nlohmann::json model_info = supported_models.get_model_info(tag_copy); - chat_engine->load_model(supported_models.get_model_path(tag_copy), model_info["default_context_length"]); + chat_engine->load_model(supported_models.get_model_path(tag_copy), model_info); current_model_tag = model_tag; } } diff --git a/src/src/main.cpp b/src/src/main.cpp index cc746b3dc..b3de64fd7 100644 --- a/src/src/main.cpp +++ b/src/src/main.cpp @@ -16,6 +16,8 @@ #include #include #include +#include + // Global variables ///@brief should_exit is used to control the server thread @@ -34,10 +36,23 @@ std::string get_executable_directory() { return "."; } +///@brief get_user_documents_directory gets the user's Documents directory +///@return the user's Documents directory path +std::string get_user_documents_directory() { + char buffer[MAX_PATH]; + if (SUCCEEDED(SHGetFolderPathA(NULL, CSIDL_PERSONAL, NULL, 0, buffer))) { + return std::string(buffer); + } + // Fallback to executable directory if Documents folder cannot be found + return get_executable_directory(); +} + ///@brief ensure_models_directory creates the models directory if it doesn't exist ///@param exe_dir the executable directory void ensure_models_directory(const std::string& exe_dir) { - std::string models_dir = exe_dir + "/models"; + // Use Documents directory for models instead of executable directory + std::string documents_dir = get_user_documents_directory(); + std::string models_dir = documents_dir + "/flm/models"; if (!std::filesystem::exists(models_dir)) { std::filesystem::create_directories(models_dir); } @@ -71,7 +86,10 @@ int main(int argc, char* argv[]) { // Initialize Unicode support for the console SetConsoleOutputCP(CP_UTF8); SetConsoleCP(CP_UTF8); - if (argc < 3 || argc > 4) { + std::string command; + std::string tag; + // Parse the command line arguments + if (argc < 2 || argc > 4) { std::cout << "Usage: " << argv[0] << " [--force]" << std::endl; std::cout << "Commands:" << std::endl; std::cout << " run - Run the model interactively" << std::endl; @@ -81,11 +99,32 @@ int main(int argc, char* argv[]) { std::cout << " --force - Force re-download even if model exists (for pull command)" << std::endl; return 1; } + command = argv[1]; + if (command == "run") { + if (argc < 3) { + std::cout << "Usage: " << argv[0] << " run " << std::endl; + return 1; + } + tag = argv[2]; + } + else if (command == "serve") { + if (argc < 3) { + tag = "llama3.2:1B"; + } + else { + tag = argv[2]; + } + } + else if (command == "pull") { + if (argc < 3) { + std::cout << "Usage: " << argv[0] << " pull " << std::endl; + return 1; + } + } + // Get the command, model tag, and force flag std::string exe_dir = get_executable_directory(); std::string config_path = exe_dir + "/model_list.json"; - std::string command = argv[1]; - std::string tag = argv[2]; bool force_redownload = false; // Check for force flag, if true, the model will be re-downloaded @@ -94,12 +133,18 @@ int main(int argc, char* argv[]) { } try { - // Load the model list - model_list supported_models(config_path, exe_dir); + // Get the Documents directory for models + std::string documents_dir = get_user_documents_directory(); + std::string models_dir = documents_dir + "/flm/models"; + + // Load the model list with Documents directory as the base + model_list supported_models(config_path, documents_dir); ModelDownloader downloader(supported_models); - // Ensure models directory exists - ensure_models_directory(exe_dir); + // Ensure models directory exists in Documents + if (!std::filesystem::exists(models_dir)) { + std::filesystem::create_directories(models_dir); + } if (command == "run") { Runner runner(supported_models, downloader, tag);