feat: replaced ollama with litellm (#18)

Co-authored-by: Sharang Parnerkar <parnerkarsharang@gmail.com>
Reviewed-on: #18

src/infrastructure/llm.rs

@@ -4,23 +4,23 @@ use dioxus::prelude::*;
 mod inner {
     use serde::{Deserialize, Serialize};
 
-    /// A single message in the OpenAI-compatible chat format used by Ollama.
+    /// A single message in the OpenAI-compatible chat format used by LiteLLM.
     #[derive(Serialize)]
     pub(super) struct ChatMessage {
         pub role: String,
         pub content: String,
     }
 
-    /// Request body for Ollama's OpenAI-compatible chat completions endpoint.
+    /// Request body for the OpenAI-compatible chat completions endpoint.
     #[derive(Serialize)]
-    pub(super) struct OllamaChatRequest {
+    pub(super) struct ChatCompletionRequest {
         pub model: String,
         pub messages: Vec<ChatMessage>,
         /// Disable streaming so we get a single JSON response.
         pub stream: bool,
     }
 
-    /// A single choice in the Ollama chat completions response.
+    /// A single choice in the chat completions response.
     #[derive(Deserialize)]
     pub(super) struct ChatChoice {
         pub message: ChatResponseMessage,
@@ -32,9 +32,9 @@ mod inner {
         pub content: String,
     }
 
-    /// Top-level response from Ollama's `/v1/chat/completions` endpoint.
+    /// Top-level response from the `/v1/chat/completions` endpoint.
     #[derive(Deserialize)]
-    pub(super) struct OllamaChatResponse {
+    pub(super) struct ChatCompletionResponse {
         pub choices: Vec<ChatChoice>,
     }
 
@@ -157,7 +157,7 @@ mod inner {
     }
 }
 
-/// Summarize an article using a local Ollama instance.
+/// Summarize an article using a LiteLLM proxy.
 ///
 /// First attempts to fetch the full article text from the provided URL.
 /// If that fails (paywall, timeout, etc.), falls back to the search snippet.
@@ -167,8 +167,8 @@ mod inner {
 ///
 /// * `snippet` - The search result snippet (fallback content)
 /// * `article_url` - The original article URL to fetch full text from
-/// * `ollama_url` - Base URL of the Ollama instance (e.g. "http://localhost:11434")
-/// * `model` - The Ollama model ID to use (e.g. "llama3.1:8b")
+/// * `litellm_url` - Base URL of the LiteLLM proxy (e.g. "http://localhost:4000")
+/// * `model` - The model ID to use (e.g. "qwen3-32b")
 ///
 /// # Returns
 ///
@@ -176,36 +176,38 @@ mod inner {
 ///
 /// # Errors
 ///
-/// Returns `ServerFnError` if the Ollama request fails or response parsing fails
+/// Returns `ServerFnError` if the LiteLLM request fails or response parsing fails
 #[post("/api/summarize")]
 pub async fn summarize_article(
     snippet: String,
     article_url: String,
-    ollama_url: String,
+    litellm_url: String,
     model: String,
 ) -> Result<String, ServerFnError> {
-    use inner::{fetch_article_text, ChatMessage, OllamaChatRequest, OllamaChatResponse};
+    use inner::{fetch_article_text, ChatCompletionRequest, ChatCompletionResponse, ChatMessage};
 
     let state: crate::infrastructure::ServerState =
         dioxus_fullstack::FullstackContext::extract().await?;
 
     // Use caller-provided values or fall back to ServerState config
-    let base_url = if ollama_url.is_empty() {
-        state.services.ollama_url.clone()
+    let base_url = if litellm_url.is_empty() {
+        state.services.litellm_url.clone()
     } else {
-        ollama_url
+        litellm_url
     };
 
     let model = if model.is_empty() {
-        state.services.ollama_model.clone()
+        state.services.litellm_model.clone()
     } else {
         model
     };
 
+    let api_key = state.services.litellm_api_key.clone();
+
     // Try to fetch the full article; fall back to the search snippet
     let article_text = fetch_article_text(&article_url).await.unwrap_or(snippet);
 
-    let request_body = OllamaChatRequest {
+    let request_body = ChatCompletionRequest {
         model,
         stream: false,
         messages: vec![ChatMessage {
@@ -223,42 +225,48 @@ pub async fn summarize_article(
 
     let url = format!("{}/v1/chat/completions", base_url.trim_end_matches('/'));
     let client = reqwest::Client::new();
-    let resp = client
+    let mut request = client
         .post(&url)
         .header("content-type", "application/json")
-        .json(&request_body)
+        .json(&request_body);
+
+    if !api_key.is_empty() {
+        request = request.header("Authorization", format!("Bearer {api_key}"));
+    }
+
+    let resp = request
         .send()
         .await
-        .map_err(|e| ServerFnError::new(format!("Ollama request failed: {e}")))?;
+        .map_err(|e| ServerFnError::new(format!("LiteLLM request failed: {e}")))?;
 
     if !resp.status().is_success() {
         let status = resp.status();
         let body = resp.text().await.unwrap_or_default();
         return Err(ServerFnError::new(format!(
-            "Ollama returned {status}: {body}"
+            "LiteLLM returned {status}: {body}"
         )));
     }
 
-    let body: OllamaChatResponse = resp
+    let body: ChatCompletionResponse = resp
         .json()
         .await
-        .map_err(|e| ServerFnError::new(format!("Failed to parse Ollama response: {e}")))?;
+        .map_err(|e| ServerFnError::new(format!("Failed to parse LiteLLM response: {e}")))?;
 
     body.choices
         .first()
         .map(|choice| choice.message.content.clone())
-        .ok_or_else(|| ServerFnError::new("Empty response from Ollama"))
+        .ok_or_else(|| ServerFnError::new("Empty response from LiteLLM"))
 }
 
 /// A lightweight chat message for the follow-up conversation.
-/// Uses simple String role ("system"/"user"/"assistant") for Ollama compatibility.
+/// Uses simple String role ("system"/"user"/"assistant") for OpenAI compatibility.
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 pub struct FollowUpMessage {
     pub role: String,
     pub content: String,
 }
 
-/// Send a follow-up question about an article using a local Ollama instance.
+/// Send a follow-up question about an article using a LiteLLM proxy.
 ///
 /// Accepts the full conversation history (system context + prior turns) and
 /// returns the assistant's next response. The system message should contain
@@ -267,8 +275,8 @@ pub struct FollowUpMessage {
 /// # Arguments
 ///
 /// * `messages` - The conversation history including system context
-/// * `ollama_url` - Base URL of the Ollama instance
-/// * `model` - The Ollama model ID to use
+/// * `litellm_url` - Base URL of the LiteLLM proxy
+/// * `model` - The model ID to use
 ///
 /// # Returns
 ///
@@ -276,30 +284,32 @@ pub struct FollowUpMessage {
 ///
 /// # Errors
 ///
-/// Returns `ServerFnError` if the Ollama request fails or response parsing fails
+/// Returns `ServerFnError` if the LiteLLM request fails or response parsing fails
 #[post("/api/chat")]
 pub async fn chat_followup(
     messages: Vec<FollowUpMessage>,
-    ollama_url: String,
+    litellm_url: String,
     model: String,
 ) -> Result<String, ServerFnError> {
-    use inner::{ChatMessage, OllamaChatRequest, OllamaChatResponse};
+    use inner::{ChatCompletionRequest, ChatCompletionResponse, ChatMessage};
 
     let state: crate::infrastructure::ServerState =
         dioxus_fullstack::FullstackContext::extract().await?;
 
-    let base_url = if ollama_url.is_empty() {
-        state.services.ollama_url.clone()
+    let base_url = if litellm_url.is_empty() {
+        state.services.litellm_url.clone()
     } else {
-        ollama_url
+        litellm_url
     };
 
     let model = if model.is_empty() {
-        state.services.ollama_model.clone()
+        state.services.litellm_model.clone()
     } else {
         model
     };
 
+    let api_key = state.services.litellm_api_key.clone();
+
     // Convert FollowUpMessage to inner ChatMessage for the request
     let chat_messages: Vec<ChatMessage> = messages
         .into_iter()
@@ -309,7 +319,7 @@ pub async fn chat_followup(
         })
         .collect();
 
-    let request_body = OllamaChatRequest {
+    let request_body = ChatCompletionRequest {
         model,
         stream: false,
         messages: chat_messages,
@@ -317,31 +327,37 @@ pub async fn chat_followup(
 
     let url = format!("{}/v1/chat/completions", base_url.trim_end_matches('/'));
     let client = reqwest::Client::new();
-    let resp = client
+    let mut request = client
         .post(&url)
         .header("content-type", "application/json")
-        .json(&request_body)
+        .json(&request_body);
+
+    if !api_key.is_empty() {
+        request = request.header("Authorization", format!("Bearer {api_key}"));
+    }
+
+    let resp = request
         .send()
         .await
-        .map_err(|e| ServerFnError::new(format!("Ollama request failed: {e}")))?;
+        .map_err(|e| ServerFnError::new(format!("LiteLLM request failed: {e}")))?;
 
     if !resp.status().is_success() {
         let status = resp.status();
         let body = resp.text().await.unwrap_or_default();
         return Err(ServerFnError::new(format!(
-            "Ollama returned {status}: {body}"
+            "LiteLLM returned {status}: {body}"
         )));
     }
 
-    let body: OllamaChatResponse = resp
+    let body: ChatCompletionResponse = resp
         .json()
         .await
-        .map_err(|e| ServerFnError::new(format!("Failed to parse Ollama response: {e}")))?;
+        .map_err(|e| ServerFnError::new(format!("Failed to parse LiteLLM response: {e}")))?;
 
     body.choices
         .first()
         .map(|choice| choice.message.content.clone())
-        .ok_or_else(|| ServerFnError::new("Empty response from Ollama"))
+        .ok_or_else(|| ServerFnError::new("Empty response from LiteLLM"))
 }
 
 #[cfg(test)]