Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4d5eedcc8b |
@@ -19,17 +19,12 @@ impl LlmClient {
|
||||
model: String,
|
||||
embed_model: String,
|
||||
) -> Self {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(300))
|
||||
.connect_timeout(std::time::Duration::from_secs(10))
|
||||
.build()
|
||||
.unwrap_or_default();
|
||||
Self {
|
||||
base_url,
|
||||
api_key,
|
||||
model,
|
||||
embed_model,
|
||||
http,
|
||||
http: reqwest::Client::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,16 +6,11 @@ use compliance_core::models::embedding::{CodeEmbedding, EmbeddingBuildRun, Embed
|
||||
use compliance_core::models::graph::CodeNode;
|
||||
use compliance_graph::graph::chunking::extract_chunks;
|
||||
use compliance_graph::graph::embedding_store::EmbeddingStore;
|
||||
use futures_util::stream::{FuturesUnordered, StreamExt};
|
||||
use tracing::{error, info};
|
||||
|
||||
use crate::error::AgentError;
|
||||
use crate::llm::LlmClient;
|
||||
|
||||
const EMBED_BATCH_SIZE: usize = 20;
|
||||
const EMBED_CONCURRENCY: usize = 4;
|
||||
const EMBED_FLUSH_EVERY: usize = 200;
|
||||
|
||||
/// RAG pipeline for building embeddings and performing retrieval
|
||||
pub struct RagPipeline {
|
||||
llm: Arc<LlmClient>,
|
||||
@@ -82,33 +77,25 @@ impl RagPipeline {
|
||||
.await
|
||||
.map_err(|e| AgentError::Other(format!("Failed to delete old embeddings: {e}")))?;
|
||||
|
||||
// Step 3: Batch embed with bounded concurrency. Flush to Mongo and
|
||||
// update progress periodically so the dashboard can show live status.
|
||||
let mut pending = Vec::with_capacity(EMBED_FLUSH_EVERY);
|
||||
// Step 3: Batch embed (small batches to stay within model limits)
|
||||
let batch_size = 20;
|
||||
let mut all_embeddings = Vec::new();
|
||||
let mut embedded_count = 0u32;
|
||||
|
||||
// Build the list of batch indices to process.
|
||||
let batches: Vec<(usize, usize)> = (0..chunks.len())
|
||||
.step_by(EMBED_BATCH_SIZE)
|
||||
.map(|start| (start, (start + EMBED_BATCH_SIZE).min(chunks.len())))
|
||||
for batch_start in (0..chunks.len()).step_by(batch_size) {
|
||||
let batch_end = (batch_start + batch_size).min(chunks.len());
|
||||
let batch_chunks = &chunks[batch_start..batch_end];
|
||||
|
||||
// Prepare texts: context_header + content
|
||||
let texts: Vec<String> = batch_chunks
|
||||
.iter()
|
||||
.map(|c| format!("{}\n{}", c.context_header, c.content))
|
||||
.collect();
|
||||
|
||||
let mut batch_iter = batches.into_iter();
|
||||
let mut in_flight = FuturesUnordered::new();
|
||||
|
||||
// Prime up to EMBED_CONCURRENCY batches.
|
||||
for _ in 0..EMBED_CONCURRENCY {
|
||||
if let Some((start, end)) = batch_iter.next() {
|
||||
in_flight.push(self.embed_batch(&chunks[start..end], start, end));
|
||||
}
|
||||
}
|
||||
|
||||
while let Some(result) = in_flight.next().await {
|
||||
match result {
|
||||
Ok((start, end, vectors)) => {
|
||||
let batch_chunks = &chunks[start..end];
|
||||
match self.llm.embed(texts).await {
|
||||
Ok(vectors) => {
|
||||
for (chunk, embedding) in batch_chunks.iter().zip(vectors) {
|
||||
pending.push(CodeEmbedding {
|
||||
all_embeddings.push(CodeEmbedding {
|
||||
id: None,
|
||||
repo_id: repo_id.to_string(),
|
||||
graph_build_id: graph_build_id.to_string(),
|
||||
@@ -126,45 +113,9 @@ impl RagPipeline {
|
||||
});
|
||||
}
|
||||
embedded_count += batch_chunks.len() as u32;
|
||||
|
||||
// Flush pending embeddings to Mongo periodically and update progress.
|
||||
if pending.len() >= EMBED_FLUSH_EVERY {
|
||||
self.embedding_store
|
||||
.store_embeddings(&pending)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
AgentError::Other(format!("Failed to store embeddings: {e}"))
|
||||
})?;
|
||||
pending.clear();
|
||||
}
|
||||
|
||||
// Always update the progress counter on the build doc — even if
|
||||
// we haven't flushed embeddings yet — so the UI shows movement.
|
||||
if let Err(e) = self
|
||||
.embedding_store
|
||||
.update_build(
|
||||
repo_id,
|
||||
graph_build_id,
|
||||
EmbeddingBuildStatus::Running,
|
||||
embedded_count,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("[{repo_id}] Failed to update build progress: {e}");
|
||||
}
|
||||
|
||||
// Queue the next batch to keep concurrency saturated.
|
||||
if let Some((s, e)) = batch_iter.next() {
|
||||
in_flight.push(self.embed_batch(&chunks[s..e], s, e));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("[{repo_id}] Embedding batch failed: {e}");
|
||||
// Flush whatever we have so partial progress isn't lost.
|
||||
if !pending.is_empty() {
|
||||
let _ = self.embedding_store.store_embeddings(&pending).await;
|
||||
}
|
||||
build.status = EmbeddingBuildStatus::Failed;
|
||||
build.error_message = Some(e.to_string());
|
||||
build.completed_at = Some(Utc::now());
|
||||
@@ -183,13 +134,11 @@ impl RagPipeline {
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: Flush any remaining embeddings
|
||||
if !pending.is_empty() {
|
||||
// Step 4: Store all embeddings
|
||||
self.embedding_store
|
||||
.store_embeddings(&pending)
|
||||
.store_embeddings(&all_embeddings)
|
||||
.await
|
||||
.map_err(|e| AgentError::Other(format!("Failed to store embeddings: {e}")))?;
|
||||
}
|
||||
|
||||
// Step 5: Update build status
|
||||
build.status = EmbeddingBuildStatus::Completed;
|
||||
@@ -212,21 +161,4 @@ impl RagPipeline {
|
||||
);
|
||||
Ok(build)
|
||||
}
|
||||
|
||||
/// Embed one batch of chunks. Returns the (start, end, vectors) tuple so
|
||||
/// out-of-order completion from `FuturesUnordered` can still be reconciled
|
||||
/// against the original chunk slice.
|
||||
async fn embed_batch(
|
||||
&self,
|
||||
batch_chunks: &[compliance_graph::graph::chunking::CodeChunk],
|
||||
start: usize,
|
||||
end: usize,
|
||||
) -> Result<(usize, usize, Vec<Vec<f64>>), AgentError> {
|
||||
let texts: Vec<String> = batch_chunks
|
||||
.iter()
|
||||
.map(|c| format!("{}\n{}", c.context_header, c.content))
|
||||
.collect();
|
||||
let vectors = self.llm.embed(texts).await?;
|
||||
Ok((start, end, vectors))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ thiserror = { workspace = true }
|
||||
|
||||
# Web-only
|
||||
reqwest = { workspace = true, optional = true }
|
||||
web-sys = { version = "0.3", optional = true, features = ["Blob", "BlobPropertyBag", "HtmlAnchorElement", "Url", "Document", "Element", "Window", "Storage", "MediaQueryList"] }
|
||||
web-sys = { version = "0.3", optional = true, features = ["Blob", "BlobPropertyBag", "HtmlAnchorElement", "Url", "Document", "Window"] }
|
||||
js-sys = { version = "0.3", optional = true }
|
||||
wasm-bindgen = { version = "0.2", optional = true }
|
||||
gloo-timers = { version = "0.3", features = ["futures"], optional = true }
|
||||
|
||||
@@ -61,77 +61,6 @@
|
||||
--ease-spring: cubic-bezier(0.34, 1.56, 0.64, 1);
|
||||
}
|
||||
|
||||
/* ── Light theme tokens ──
|
||||
Applied when the user has explicitly chosen light (`data-theme="light"`)
|
||||
OR when their OS prefers light AND they have made no explicit choice. */
|
||||
:root[data-theme="light"] {
|
||||
--bg-primary: #f5f7fb;
|
||||
--bg-secondary: #ffffff;
|
||||
--bg-card: rgba(255, 255, 255, 0.85);
|
||||
--bg-card-solid: #ffffff;
|
||||
--bg-card-hover: #f1f5fb;
|
||||
--bg-elevated: #f8fafc;
|
||||
|
||||
--text-primary: #0c1426;
|
||||
--text-secondary: #475569;
|
||||
--text-tertiary: #8a9bb4;
|
||||
|
||||
--accent: #0070d4;
|
||||
--accent-hover: #0080f0;
|
||||
--accent-muted: rgba(0, 112, 212, 0.10);
|
||||
--accent-glow: 0 0 20px rgba(0, 112, 212, 0.10);
|
||||
|
||||
--border: #e2e8f0;
|
||||
--border-bright: #cbd5e1;
|
||||
--border-accent: rgba(0, 112, 212, 0.30);
|
||||
|
||||
--danger: #dc2626;
|
||||
--danger-bg: rgba(220, 38, 38, 0.08);
|
||||
--warning: #d97706;
|
||||
--warning-bg: rgba(217, 119, 6, 0.08);
|
||||
--success: #16a34a;
|
||||
--success-bg: rgba(22, 163, 74, 0.08);
|
||||
--info: #2563eb;
|
||||
--info-bg: rgba(37, 99, 235, 0.08);
|
||||
--orange: #ea580c;
|
||||
--orange-bg: rgba(234, 88, 12, 0.08);
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: light) {
|
||||
:root:not([data-theme="dark"]) {
|
||||
--bg-primary: #f5f7fb;
|
||||
--bg-secondary: #ffffff;
|
||||
--bg-card: rgba(255, 255, 255, 0.85);
|
||||
--bg-card-solid: #ffffff;
|
||||
--bg-card-hover: #f1f5fb;
|
||||
--bg-elevated: #f8fafc;
|
||||
|
||||
--text-primary: #0c1426;
|
||||
--text-secondary: #475569;
|
||||
--text-tertiary: #8a9bb4;
|
||||
|
||||
--accent: #0070d4;
|
||||
--accent-hover: #0080f0;
|
||||
--accent-muted: rgba(0, 112, 212, 0.10);
|
||||
--accent-glow: 0 0 20px rgba(0, 112, 212, 0.10);
|
||||
|
||||
--border: #e2e8f0;
|
||||
--border-bright: #cbd5e1;
|
||||
--border-accent: rgba(0, 112, 212, 0.30);
|
||||
|
||||
--danger: #dc2626;
|
||||
--danger-bg: rgba(220, 38, 38, 0.08);
|
||||
--warning: #d97706;
|
||||
--warning-bg: rgba(217, 119, 6, 0.08);
|
||||
--success: #16a34a;
|
||||
--success-bg: rgba(22, 163, 74, 0.08);
|
||||
--info: #2563eb;
|
||||
--info-bg: rgba(37, 99, 235, 0.08);
|
||||
--orange: #ea580c;
|
||||
--orange-bg: rgba(234, 88, 12, 0.08);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ── Reset & Base ── */
|
||||
|
||||
@@ -467,44 +396,6 @@ code {
|
||||
background: rgba(0, 200, 255, 0.06);
|
||||
}
|
||||
|
||||
.theme-toggle {
|
||||
background: none;
|
||||
border: none;
|
||||
border-top: 1px solid var(--border);
|
||||
color: var(--text-secondary);
|
||||
padding: 11px 18px;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 11px;
|
||||
font-family: var(--font-body);
|
||||
font-size: 13.5px;
|
||||
font-weight: 500;
|
||||
transition: color 0.2s, background 0.2s;
|
||||
width: 100%;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.theme-toggle:hover {
|
||||
color: var(--accent);
|
||||
background: var(--accent-muted);
|
||||
}
|
||||
|
||||
.theme-toggle svg {
|
||||
flex-shrink: 0;
|
||||
opacity: 0.75;
|
||||
transition: opacity 0.2s;
|
||||
}
|
||||
|
||||
.theme-toggle:hover svg {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.sidebar.collapsed .theme-toggle {
|
||||
justify-content: center;
|
||||
padding: 11px 0;
|
||||
}
|
||||
|
||||
.sidebar.collapsed .sidebar-header {
|
||||
padding: 22px 0;
|
||||
justify-content: center;
|
||||
@@ -3998,33 +3889,3 @@ tbody tr:last-child td {
|
||||
.copyable code, .copyable .mono { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.code-snippet-wrapper { position: relative; }
|
||||
.code-snippet-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: 4px; gap: 8px; }
|
||||
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════
|
||||
LIGHT THEME — surface overrides for the few hardcoded dark
|
||||
colors that don't go through CSS custom properties.
|
||||
═══════════════════════════════════════════════════════════════ */
|
||||
|
||||
:root[data-theme="light"] .main-content {
|
||||
background-image: radial-gradient(circle at 1px 1px, rgba(100, 116, 139, 0.18) 1px, transparent 0);
|
||||
}
|
||||
:root[data-theme="light"] .code-block {
|
||||
background: #f8fafc;
|
||||
color: #0c1426;
|
||||
}
|
||||
:root[data-theme="light"] .graph-stab-overlay {
|
||||
background: radial-gradient(ellipse at center, rgba(245, 247, 251, 0.92) 0%, rgba(245, 247, 251, 0.98) 100%);
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: light) {
|
||||
:root:not([data-theme="dark"]) .main-content {
|
||||
background-image: radial-gradient(circle at 1px 1px, rgba(100, 116, 139, 0.18) 1px, transparent 0);
|
||||
}
|
||||
:root:not([data-theme="dark"]) .code-block {
|
||||
background: #f8fafc;
|
||||
color: #0c1426;
|
||||
}
|
||||
:root:not([data-theme="dark"]) .graph-stab-overlay {
|
||||
background: radial-gradient(ellipse at center, rgba(245, 247, 251, 0.92) 0%, rgba(245, 247, 251, 0.98) 100%);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,5 +12,4 @@ pub mod pentest_wizard;
|
||||
pub mod severity_badge;
|
||||
pub mod sidebar;
|
||||
pub mod stat_card;
|
||||
pub mod theme_toggle;
|
||||
pub mod toast;
|
||||
|
||||
@@ -4,7 +4,6 @@ use dioxus_free_icons::icons::bs_icons::*;
|
||||
use dioxus_free_icons::Icon;
|
||||
|
||||
use crate::app::Route;
|
||||
use crate::components::theme_toggle::ThemeToggle;
|
||||
|
||||
struct NavItem {
|
||||
label: &'static str,
|
||||
@@ -107,7 +106,6 @@ pub fn Sidebar() -> Element {
|
||||
}
|
||||
// Spacer pushes footer to the bottom
|
||||
div { class: "sidebar-spacer" }
|
||||
ThemeToggle { collapsed: collapsed() }
|
||||
button {
|
||||
class: "sidebar-toggle",
|
||||
onclick: move |_| collapsed.set(!collapsed()),
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
use dioxus::prelude::*;
|
||||
use dioxus_free_icons::icons::bs_icons::{BsMoonStars, BsSun};
|
||||
use dioxus_free_icons::Icon;
|
||||
|
||||
#[cfg(feature = "web")]
|
||||
const STORAGE_KEY: &str = "compliance-scanner.theme";
|
||||
|
||||
/// Sidebar-footer theme toggle. Reads the initial state on mount from
|
||||
/// localStorage (explicit user choice) or `prefers-color-scheme` (OS default),
|
||||
/// then writes back to both the `<html data-theme="...">` attribute and
|
||||
/// localStorage on every click.
|
||||
#[component]
|
||||
pub fn ThemeToggle(collapsed: bool) -> Element {
|
||||
// `None` until the on-mount effect resolves the real value, so SSR doesn't
|
||||
// render the wrong icon for the user's actual theme.
|
||||
let mut is_dark = use_signal(|| None::<bool>);
|
||||
|
||||
use_effect(move || {
|
||||
let (dark, from_storage) = initial_theme();
|
||||
is_dark.set(Some(dark));
|
||||
// If the user already made an explicit choice (in localStorage), assert it
|
||||
// on the DOM so an OS-vs-stored mismatch can't briefly show the wrong theme.
|
||||
if from_storage {
|
||||
apply_theme(dark);
|
||||
}
|
||||
});
|
||||
|
||||
let label = if collapsed {
|
||||
""
|
||||
} else if is_dark().unwrap_or(true) {
|
||||
"Light mode"
|
||||
} else {
|
||||
"Dark mode"
|
||||
};
|
||||
|
||||
let title = if is_dark().unwrap_or(true) {
|
||||
"Switch to light mode"
|
||||
} else {
|
||||
"Switch to dark mode"
|
||||
};
|
||||
|
||||
rsx! {
|
||||
button {
|
||||
class: "theme-toggle",
|
||||
r#type: "button",
|
||||
title: "{title}",
|
||||
"aria-label": "{title}",
|
||||
onclick: move |_| {
|
||||
let next_dark = !is_dark().unwrap_or(true);
|
||||
is_dark.set(Some(next_dark));
|
||||
apply_theme(next_dark);
|
||||
},
|
||||
if is_dark().unwrap_or(true) {
|
||||
Icon { icon: BsSun, width: 16, height: 16 }
|
||||
} else {
|
||||
Icon { icon: BsMoonStars, width: 16, height: 16 }
|
||||
}
|
||||
if !collapsed {
|
||||
span { class: "theme-toggle-label", "{label}" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `(is_dark, from_storage)`. `from_storage` is true when an explicit
|
||||
/// user choice is in localStorage; false when we fell back to OS preference
|
||||
/// (or to the dark default).
|
||||
#[cfg(feature = "web")]
|
||||
fn initial_theme() -> (bool, bool) {
|
||||
if let Some(window) = web_sys::window() {
|
||||
if let Ok(Some(storage)) = window.local_storage() {
|
||||
if let Ok(Some(value)) = storage.get_item(STORAGE_KEY) {
|
||||
return (value == "dark", true);
|
||||
}
|
||||
}
|
||||
if let Ok(Some(mql)) = window.match_media("(prefers-color-scheme: dark)") {
|
||||
return (mql.matches(), false);
|
||||
}
|
||||
}
|
||||
(true, false)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "web"))]
|
||||
fn initial_theme() -> (bool, bool) {
|
||||
(true, false)
|
||||
}
|
||||
|
||||
#[cfg(feature = "web")]
|
||||
fn apply_theme(dark: bool) {
|
||||
let theme = if dark { "dark" } else { "light" };
|
||||
if let Some(window) = web_sys::window() {
|
||||
if let Some(document) = window.document() {
|
||||
if let Some(root) = document.document_element() {
|
||||
let _ = root.set_attribute("data-theme", theme);
|
||||
}
|
||||
}
|
||||
if let Ok(Some(storage)) = window.local_storage() {
|
||||
let _ = storage.set_item(STORAGE_KEY, theme);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "web"))]
|
||||
fn apply_theme(_dark: bool) {}
|
||||
Reference in New Issue
Block a user