add llama provider (#37)

bot_microservice/constants.py

@@ -54,6 +54,7 @@ class ChatGptModelsEnum(StrEnum):
     gpt_3_5_turbo_stream_Cromicle = "gpt-3.5-turbo-stream-Cromicle"
     gpt_4_stream_Chatgpt4Online = "gpt-4-stream-Chatgpt4Online"
     gpt_3_5_turbo_stream_gptalk = "gpt-3.5-turbo-stream-gptalk"
+    llama2 = "llama2"
     gpt_3_5_turbo_stream_ChatgptDemo = "gpt-3.5-turbo-stream-ChatgptDemo"
     gpt_3_5_turbo_stream_gptforlove = "gpt-3.5-turbo-stream-gptforlove"
 

chatgpt_microservice/include/free_gpt.h

@@ -36,6 +36,7 @@ public:
     boost::asio::awaitable<void> gptalk(std::shared_ptr<Channel>, nlohmann::json);
     boost::asio::awaitable<void> gptForLove(std::shared_ptr<Channel>, nlohmann::json);
     boost::asio::awaitable<void> chatGptDemo(std::shared_ptr<Channel>, nlohmann::json);
+    boost::asio::awaitable<void> llama2(std::shared_ptr<Channel>, nlohmann::json);
 
 private:
     boost::asio::awaitable<std::expected<boost::beast::ssl_stream<boost::beast::tcp_stream>, std::string>>

chatgpt_microservice/src/free_gpt.cpp

@@ -575,6 +575,90 @@ std::expected<nlohmann::json, std::string> callZeus(const std::string& host, con
     return rsp;
 }
 
+struct CurlHttpRequest {
+    CURL* curl;
+    std::string url;
+    std::string_view http_proxy;
+    size_t (*cb)(void* contents, size_t size, size_t nmemb, void* userp);
+    void* input;
+    std::unordered_map<std::string, std::string>& headers;
+    std::string body;
+    std::multimap<std::string, std::string>* response_header_ptr{nullptr};
+    int32_t expect_response_code{200};
+    bool ssl_verify{false};
+};
+
+std::optional<std::string> sendHttpRequest(const CurlHttpRequest& curl_http_request) {
+    auto& [curl, url, http_proxy, stream_action_cb, input, http_headers, body, response_header_ptr, response_code,
+           ssl_verify] = curl_http_request;
+    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+    if (!http_proxy.empty())
+        curl_easy_setopt(curl, CURLOPT_PROXY, http_proxy.data());
+    if (!ssl_verify) {
+        curl_easy_setopt(curl, CURLOPT_CAINFO, nullptr);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
+    }
+    curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 20L);
+    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 120L);
+    curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 30L);
+    if (stream_action_cb != nullptr)
+        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, stream_action_cb);
+    if (input != nullptr)
+        curl_easy_setopt(curl, CURLOPT_WRITEDATA, input);
+
+    std::string buffer;
+    auto header_callback = [](char* buffer, size_t size, size_t nitems, void* userdata) {
+        std::string str{(char*)buffer, size * nitems};
+        static_cast<std::string*>(userdata)->append((char*)buffer, size * nitems);
+        return nitems * size;
+    };
+    size_t (*fn_header_callback)(char* buffer, size_t size, size_t nitems, void* userdata) = header_callback;
+    if (response_header_ptr) {
+        curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, fn_header_callback);
+        curl_easy_setopt(curl, CURLOPT_HEADERDATA, &buffer);
+    }
+
+    struct curl_slist* headers = nullptr;
+    std::vector<std::string> headers_list;
+    for (auto& [k, v] : http_headers)
+        headers_list.emplace_back(std::format("{}: {}", k, v));
+    for (auto& header : headers_list)
+        headers = curl_slist_append(headers, header.c_str());
+    if (headers != nullptr)
+        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+
+    ScopeExit auto_exit{[=] { curl_slist_free_all(headers); }};
+
+    if (!body.empty())
+        curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str());
+
+    auto res = curl_easy_perform(curl);
+    if (res != CURLE_OK) {
+        auto error_info = std::format("curl_easy_perform() failed:{}", curl_easy_strerror(res));
+        return error_info;
+    }
+    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
+    if (response_code != 200) {
+        auto error_info = std::format("response_code :{}", response_code);
+        return error_info;
+    }
+    if (!buffer.empty() && response_header_ptr != nullptr) {
+        std::regex pattern(R"(([^:\r\n]+):([^\r\n]+))");
+        std::smatch matches;
+        auto start = buffer.cbegin();
+        auto end = buffer.cend();
+        while (std::regex_search(start, end, matches, pattern)) {
+            std::string field_name = matches[1].str();
+            std::string field_value = matches[2].str();
+            (*response_header_ptr).insert(std::pair{field_name, field_value});
+            start = matches[0].second;
+        }
+    }
+    return std::nullopt;
+}
+
 }  // namespace
 
 FreeGpt::FreeGpt(Config& cfg)
@@ -2558,15 +2642,20 @@ boost::asio::awaitable<void> FreeGpt::chatGptDemo(std::shared_ptr<Channel> ch, n
 
     auto prompt = json.at("meta").at("content").at("parts").at(0).at("content").get<std::string>();
 
-    CURLcode res;
-    int32_t response_code;
-
     struct Input {
         std::shared_ptr<Channel> ch;
         std::string recv;
     };
     Input input;
 
+    std::unordered_map<std::string, std::string> http_headers{
+        {"authority", "chat.chatgptdemo.net"},
+        {"origin", "https://chat.chatgptdemo.net"},
+        {"referer", "https://chat.chatgptdemo.net/"},
+    };
+    std::multimap<std::string, std::string> response_header;
+
+    CURLcode res;
     CURL* curl = curl_easy_init();
     if (!curl) {
         auto error_info = std::format("curl_easy_init() failed:{}", curl_easy_strerror(res));
@@ -2574,48 +2663,39 @@ boost::asio::awaitable<void> FreeGpt::chatGptDemo(std::shared_ptr<Channel> ch, n
         ch->try_send(err, error_info);
         co_return;
     }
-    curl_easy_setopt(curl, CURLOPT_URL, "https://chat.chatgptdemo.net/");
+    ScopeExit auto_exit{[=] { curl_easy_cleanup(curl); }};
 
-    if (!m_cfg.http_proxy.empty())
+    auto ret = sendHttpRequest(CurlHttpRequest{
-        curl_easy_setopt(curl, CURLOPT_PROXY, m_cfg.http_proxy.c_str());
+        .curl = curl,
-    curlEasySetopt(curl);
+        .url = "https://chat.chatgptdemo.net/",
-
+        .http_proxy = m_cfg.http_proxy,
-    auto action_cb = [](void* contents, size_t size, size_t nmemb, void* userp) -> size_t {
+        .cb = [](void* contents, size_t size, size_t nmemb, void* userp) mutable -> size_t {
-        auto input_ptr = static_cast<Input*>(userp);
+            auto input_ptr = static_cast<Input*>(userp);
-        std::string data{(char*)contents, size * nmemb};
+            std::string data{(char*)contents, size * nmemb};
-        auto& [ch, recv] = *input_ptr;
+            auto& [ch, recv] = *input_ptr;
-        recv.append(data);
+            recv.append(data);
-        return size * nmemb;
+            return size * nmemb;
-    };
+        },
-    size_t (*action_fn)(void* contents, size_t size, size_t nmemb, void* userp) = action_cb;
+        .input = [&] -> void* {
-    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, action_fn);
+            input.recv.clear();
-    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &input);
+            return &input;
-
+        }(),
-    struct curl_slist* headers = nullptr;
+        .headers = http_headers,
-    headers = curl_slist_append(headers, "origin: https://chat.chatgptdemo.net");
+        .body = std::string{},
-    headers = curl_slist_append(headers, "referer: https://chat.chatgptdemo.net/");
+        .response_header_ptr = &response_header,
-    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+        .expect_response_code = 200,
-
+        .ssl_verify = false,
-    ScopeExit auto_exit{[=] {
+    });
-        curl_slist_free_all(headers);
+    if (ret) {
-        curl_easy_cleanup(curl);
-    }};
-
-    res = curl_easy_perform(curl);
-    if (res != CURLE_OK) {
         co_await boost::asio::post(boost::asio::bind_executor(ch->get_executor(), boost::asio::use_awaitable));
-        auto error_info = std::format("curl_easy_perform() failed:{}", curl_easy_strerror(res));
+        ch->try_send(err, ret.value());
-        ch->try_send(err, error_info);
         co_return;
     }
-    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
+    for (auto& [k, v] : response_header)
-    if (response_code != 200) {
+        SPDLOG_INFO("{}: {}", k, v);
-        co_await boost::asio::post(boost::asio::bind_executor(ch->get_executor(), boost::asio::use_awaitable));
+
-        ch->try_send(err, std::format("chatGptDemo http code:{}", response_code));
+    auto match_ret = findAll(R"(<div id="USERID" style="display: none">(.*)<\/div>)", input.recv);
-        co_return;
+    if (match_ret.empty()) {
-    }
-    auto ret = findAll(R"(<div id="USERID" style="display: none">(.*)<\/div>)", input.recv);
-    if (ret.empty()) {
         co_await boost::asio::post(boost::asio::bind_executor(ch->get_executor(), boost::asio::use_awaitable));
         ch->try_send(err, std::format("not found userid"));
         co_return;
@@ -2626,119 +2706,188 @@ boost::asio::awaitable<void> FreeGpt::chatGptDemo(std::shared_ptr<Channel> ch, n
         std::string value = str.substr(start, end - start);
         return value;
     };
-    auto user_id = extract_value(ret[0]);
+    auto user_id = extract_value(match_ret[0]);
     SPDLOG_INFO("user_id: [{}]", user_id);
 
-    curl_easy_setopt(curl, CURLOPT_URL, "https://chat.chatgptdemo.net/new_chat");
+    ret = sendHttpRequest(CurlHttpRequest{
-
+        .curl = curl,
-    if (!m_cfg.http_proxy.empty())
+        .url = "https://chat.chatgptdemo.net/new_chat",
-        curl_easy_setopt(curl, CURLOPT_PROXY, m_cfg.http_proxy.c_str());
+        .http_proxy = m_cfg.http_proxy,
-    input.recv.clear();
+        .cb = [](void* contents, size_t size, size_t nmemb, void* userp) mutable -> size_t {
-    auto api_action_cb = [](void* contents, size_t size, size_t nmemb, void* userp) -> size_t {
+            auto input_ptr = static_cast<Input*>(userp);
-        auto input_ptr = static_cast<Input*>(userp);
+            std::string data{(char*)contents, size * nmemb};
-        std::string data{(char*)contents, size * nmemb};
+            auto& [ch, recv] = *input_ptr;
-        auto& [ch, recv] = *input_ptr;
+            recv.append(data);
-        recv.append(data);
+            return size * nmemb;
-        return size * nmemb;
+        },
-    };
+        .input = [&] -> void* {
-    size_t (*api_cb)(void* contents, size_t size, size_t nmemb, void* userp) = api_action_cb;
+            input.recv.clear();
-    curlEasySetopt(curl);
+            input.ch = ch;
-    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, api_cb);
+            return &input;
-    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &input);
+        }(),
-
+        .headers = [&] -> auto& {
-    constexpr std::string_view json_str = R"({"user_id":"user_id"})";
+            http_headers.emplace("Content-Type", "application/json");
-    nlohmann::json request = nlohmann::json::parse(json_str, nullptr, false);
+            return http_headers;
-    request["user_id"] = user_id;
+        }(),
-    auto request_str = request.dump();
+        .body = [&] -> std::string {
-    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, request_str.c_str());
+            constexpr std::string_view json_str = R"({"user_id":"user_id"})";
-
+            nlohmann::json request = nlohmann::json::parse(json_str, nullptr, false);
-    headers = curl_slist_append(headers, "Content-Type: application/json");
+            request["user_id"] = user_id;
-    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+            return request.dump();
-
+        }(),
-    res = curl_easy_perform(curl);
+        .response_header_ptr = nullptr,
-    if (res != CURLE_OK) {
+        .expect_response_code = 200,
+        .ssl_verify = false,
+    });
+    if (ret) {
         co_await boost::asio::post(boost::asio::bind_executor(ch->get_executor(), boost::asio::use_awaitable));
-        auto error_info = std::format("curl_easy_perform() failed:{}", curl_easy_strerror(res));
+        ch->try_send(err, ret.value());
-        ch->try_send(err, error_info);
-        co_return;
-    }
-    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
-    if (response_code != 200) {
-        co_await boost::asio::post(boost::asio::bind_executor(ch->get_executor(), boost::asio::use_awaitable));
-        ch->try_send(err, std::format("chatGptDemo http code:{}", response_code));
         co_return;
     }
+
     SPDLOG_INFO("input.recv: [{}]", input.recv);
     nlohmann::json get_text_rsp = nlohmann::json::parse(input.recv, nullptr, false);
     auto chat_id = get_text_rsp["id_"].get<std::string>();
     SPDLOG_INFO("chat_id: [{}]", chat_id);
-    input.recv.clear();
 
-    curl_easy_setopt(curl, CURLOPT_URL, "https://chat.chatgptdemo.net/chat_api_stream");
+    ret = sendHttpRequest(CurlHttpRequest{
-    if (!m_cfg.http_proxy.empty())
+        .curl = curl,
-        curl_easy_setopt(curl, CURLOPT_PROXY, m_cfg.http_proxy.c_str());
+        .url = "https://chat.chatgptdemo.net/chat_api_stream",
-    curlEasySetopt(curl);
+        .http_proxy = m_cfg.http_proxy,
-
+        .cb = [](void* contents, size_t size, size_t nmemb, void* userp) mutable -> size_t {
-    auto stream_action_cb = [](void* contents, size_t size, size_t nmemb, void* userp) mutable -> size_t {
+            auto input_ptr = static_cast<Input*>(userp);
-        auto input_ptr = static_cast<Input*>(userp);
+            std::string data{(char*)contents, size * nmemb};
-        std::string data{(char*)contents, size * nmemb};
+            auto& [ch, recv] = *input_ptr;
-        auto& [ch, recv] = *input_ptr;
+            recv.append(data);
-        recv.append(data);
+            while (true) {
-        while (true) {
+                auto position = recv.find("\n");
-            auto position = recv.find("\n");
+                if (position == std::string::npos)
-            if (position == std::string::npos)
+                    break;
-                break;
+                auto msg = recv.substr(0, position + 1);
-            auto msg = recv.substr(0, position + 1);
+                recv.erase(0, position + 1);
-            recv.erase(0, position + 1);
+                msg.pop_back();
-            msg.pop_back();
+                if (msg.empty() || !msg.contains("content"))
-            if (msg.empty() || !msg.contains("content"))
+                    continue;
-                continue;
+                if (!msg.starts_with("data: "))
-            if (!msg.starts_with("data: "))
+                    continue;
-                continue;
+                msg.erase(0, 6);
-            msg.erase(0, 6);
+                boost::system::error_code err{};
-            boost::system::error_code err{};
+                nlohmann::json line_json = nlohmann::json::parse(msg, nullptr, false);
-            nlohmann::json line_json = nlohmann::json::parse(msg, nullptr, false);
+                if (line_json.is_discarded()) {
-            if (line_json.is_discarded()) {
+                    SPDLOG_ERROR("json parse error: [{}]", msg);
-                SPDLOG_ERROR("json parse error: [{}]", msg);
+                    boost::asio::post(ch->get_executor(),
-                boost::asio::post(ch->get_executor(),
+                                      [=] { ch->try_send(err, std::format("json parse error: [{}]", msg)); });
-                                  [=] { ch->try_send(err, std::format("json parse error: [{}]", msg)); });
+                    continue;
-                continue;
+                }
+                auto str = line_json["choices"][0]["delta"]["content"].get<std::string>();
+                if (!str.empty())
+                    boost::asio::post(ch->get_executor(), [=] { ch->try_send(err, str); });
             }
-            auto str = line_json["choices"][0]["delta"]["content"].get<std::string>();
+            return size * nmemb;
-            if (!str.empty())
+        },
-                boost::asio::post(ch->get_executor(), [=] { ch->try_send(err, str); });
+        .input = [&] -> void* {
-        }
+            input.recv.clear();
-        return size * nmemb;
+            input.ch = ch;
-    };
+            return &input;
-    size_t (*stream_action_cb_fn)(void* contents, size_t size, size_t nmemb, void* userp) = stream_action_cb;
+        }(),
-    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, stream_action_cb_fn);
+        .headers = http_headers,
-    input.ch = ch;
+        .body = [&] -> std::string {
-    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &input);
+            constexpr std::string_view ask_json_str =
-    headers = curl_slist_append(headers, "authority: chat.chatgptdemo.net");
+                R"({"question": "hello", "chat_id": "6524f3640d0d824902f598c1", "timestamp": 1696920420510})";
-    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+            nlohmann::json ask_request = nlohmann::json::parse(ask_json_str, nullptr, false);
-
+            ask_request["question"] = prompt;
-    constexpr std::string_view ask_json_str =
+            ask_request["chat_id"] = chat_id;
-        R"({"question": "hello", "chat_id": "6524f3640d0d824902f598c1", "timestamp": 1696920420510})";
+            uint64_t timestamp = getTimestamp();
-    nlohmann::json ask_request = nlohmann::json::parse(ask_json_str, nullptr, false);
+            ask_request["timestamp"] = timestamp;
-    ask_request["question"] = prompt;
+            std::string ask_request_str = ask_request.dump();
-    ask_request["chat_id"] = chat_id;
+            SPDLOG_INFO("ask_request_str: [{}]", ask_request_str);
-    uint64_t timestamp = getTimestamp();
+            return ask_request_str;
-    request["timestamp"] = timestamp;
+        }(),
-    std::string ask_request_str = ask_request.dump();
+        .response_header_ptr = nullptr,
-    SPDLOG_INFO("ask_request_str: [{}]", ask_request_str);
+        .expect_response_code = 200,
-    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, ask_request_str.c_str());
+        .ssl_verify = false,
-
+    });
-    res = curl_easy_perform(curl);
+    if (ret) {
-    if (res != CURLE_OK) {
         co_await boost::asio::post(boost::asio::bind_executor(ch->get_executor(), boost::asio::use_awaitable));
-        auto error_info = std::format("curl_easy_perform() failed:{}", curl_easy_strerror(res));
+        ch->try_send(err, ret.value());
-        ch->try_send(err, error_info);
+        co_return;
-        co_return;
+    }
-    }
+    co_return;
-    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
+}
-    if (response_code != 200) {
+
-        co_await boost::asio::post(boost::asio::bind_executor(ch->get_executor(), boost::asio::use_awaitable));
+boost::asio::awaitable<void> FreeGpt::llama2(std::shared_ptr<Channel> ch, nlohmann::json json) {
-        ch->try_send(err, std::format("gptalk http code:{}", response_code));
+    co_await boost::asio::post(boost::asio::bind_executor(*m_thread_pool_ptr, boost::asio::use_awaitable));
+    ScopeExit _exit{[=] { boost::asio::post(ch->get_executor(), [=] { ch->close(); }); }};
+    boost::system::error_code err{};
+
+    auto prompt = json.at("meta").at("content").at("parts").at(0).at("content").get<std::string>();
+
+    struct Input {
+        std::shared_ptr<Channel> ch;
+        std::string recv;
+    };
+    Input input;
+
+    CURLcode res;
+    CURL* curl = curl_easy_init();
+    if (!curl) {
+        auto error_info = std::format("curl_easy_init() failed:{}", curl_easy_strerror(res));
+        co_await boost::asio::post(boost::asio::bind_executor(ch->get_executor(), boost::asio::use_awaitable));
+        ch->try_send(err, error_info);
+        co_return;
+    }
+    ScopeExit auto_exit{[=] { curl_easy_cleanup(curl); }};
+
+    auto ret = sendHttpRequest(CurlHttpRequest{
+        .curl = curl,
+        .url = "https://www.llama2.ai/api",
+        .http_proxy = m_cfg.http_proxy,
+        .cb = [](void* contents, size_t size, size_t nmemb, void* userp) mutable -> size_t {
+            auto input_ptr = static_cast<Input*>(userp);
+            std::string data{(char*)contents, size * nmemb};
+            auto& [ch, recv] = *input_ptr;
+            boost::asio::post(ch->get_executor(), [=] {
+                boost::system::error_code err{};
+                ch->try_send(err, data);
+            });
+            return size * nmemb;
+        },
+        .input = [&] -> void* {
+            input.recv.clear();
+            input.ch = ch;
+            return &input;
+        }(),
+        .headers = [&] -> auto& {
+            static std::unordered_map<std::string, std::string> headers{
+                {"Accept", "*/*"},
+                {"origin", "https://www.llama2.ai"},
+                {"referer", "https://www.llama2.ai/"},
+                {"Content-Type", "text/plain;charset=UTF-8"},
+            };
+            return headers;
+        }(),
+        .body = [&] -> std::string {
+            constexpr std::string_view ask_json_str = R"({
+                "prompt": "[INST] hello [/INST]\n",
+                "version": "2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf",
+                "systemPrompt": "You are a helpful assistant.",
+                "temperature": 0.75,
+                "topP": 0.9,
+                "maxTokens": 800
+            })";
+            nlohmann::json ask_request = nlohmann::json::parse(ask_json_str, nullptr, false);
+            ask_request["prompt"] = std::format("[INST] {} [/INST]\n", prompt);
+            std::string ask_request_str = ask_request.dump();
+            SPDLOG_INFO("ask_request_str: [{}]", ask_request_str);
+            return ask_request_str;
+        }(),
+        .response_header_ptr = nullptr,
+        .expect_response_code = 200,
+        .ssl_verify = false,
+    });
+    if (ret) {
+        co_await boost::asio::post(boost::asio::bind_executor(ch->get_executor(), boost::asio::use_awaitable));
+        ch->try_send(err, ret.value());
         co_return;
     }
     co_return;

chatgpt_microservice/src/main.cpp

@@ -351,6 +351,7 @@ int main(int argc, char** argv) {
     ADD_METHOD("gpt-3.5-turbo-stream-ChatForAi", FreeGpt::chatForAi);
     ADD_METHOD("gpt-3.5-turbo-stream-gptforlove", FreeGpt::gptForLove);
     ADD_METHOD("gpt-3.5-turbo-stream-ChatgptDemo", FreeGpt::chatGptDemo);
+    ADD_METHOD("llama2", FreeGpt::llama2);
 
     SPDLOG_INFO("active provider:");
     for (auto& [provider, _] : gpt_function)