Merge origin/main (Tantivy content search) into delta-sync branch

Both sides added a parameter to create_application_services and a
setup step before it: this branch's storage-usage/quota service (for
the instant-upload path) and main's Tantivy content index (for
SearchService). The resolution keeps both — the signature takes both
arguments and the build runs storage usage as step 3c and the content
index as 3d.

https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS
This commit is contained in:
Claude
2026-06-11 18:32:27 +00:00
19 changed files with 2807 additions and 68 deletions
+68
View File
@@ -898,6 +898,44 @@ impl Default for FeaturesConfig {
}
}
/// Content-search configuration (embedded Tantivy index over file names and
/// extracted file content).
///
/// The index is a derived artifact fed by a background worker on the
/// maintenance pool — none of these knobs affect request-path latency.
#[derive(Debug, Clone)]
pub struct ContentSearchConfig {
/// Master switch. When disabled, search falls back to name-only SQL and
/// a janitor keeps the (always-installed) dirty queue empty.
/// Env: `OXICLOUD_ENABLE_CONTENT_SEARCH`.
pub enabled: bool,
/// Index directory. Default: `{storage_path}/.search-index`.
/// Env: `OXICLOUD_CONTENT_INDEX_DIR`.
pub index_dir: Option<PathBuf>,
/// Worker drain cadence in milliseconds — the upper bound on how long a
/// new upload takes to become content-searchable. Default: 1500.
/// Env: `OXICLOUD_CONTENT_INDEX_FLUSH_MS`.
pub flush_interval_ms: u64,
/// Files larger than this are indexed by NAME only (no text extraction).
/// Default: 32 MiB. Env: `OXICLOUD_CONTENT_INDEX_MAX_FILE_BYTES`.
pub max_extract_file_bytes: u64,
/// Hard cap on extracted text per blob fed to the index. Default: 1 MiB.
/// Env: `OXICLOUD_CONTENT_INDEX_MAX_TEXT_BYTES`.
pub max_text_bytes: usize,
}
impl Default for ContentSearchConfig {
fn default() -> Self {
Self {
enabled: true,
index_dir: None,
flush_interval_ms: 1500,
max_extract_file_bytes: 32 * 1024 * 1024,
max_text_bytes: 1024 * 1024,
}
}
}
/// Global application configuration
#[derive(Debug, Clone)]
pub struct AppConfig {
@@ -937,6 +975,8 @@ pub struct AppConfig {
pub magic_link: MagicLinkConfig,
/// I18n configuration (default locale for server-rendered surfaces)
pub i18n: I18nConfig,
/// Content-search configuration (embedded full-text index)
pub content_search: ContentSearchConfig,
}
/// Server-side i18n knobs.
@@ -988,6 +1028,7 @@ impl Default for AppConfig {
smtp: SmtpConfig::default(),
magic_link: MagicLinkConfig::default(),
i18n: I18nConfig::default(),
content_search: ContentSearchConfig::default(),
}
}
}
@@ -1250,6 +1291,33 @@ impl AppConfig {
config.features.enable_music = val;
}
// Content search (embedded Tantivy index)
if let Ok(v) = env::var("OXICLOUD_ENABLE_CONTENT_SEARCH").map(|v| v.parse::<bool>())
&& let Ok(val) = v
{
config.content_search.enabled = val;
}
if let Ok(dir) = env::var("OXICLOUD_CONTENT_INDEX_DIR")
&& !dir.trim().is_empty()
{
config.content_search.index_dir = Some(PathBuf::from(dir.trim()));
}
if let Ok(v) = env::var("OXICLOUD_CONTENT_INDEX_FLUSH_MS").map(|v| v.parse::<u64>())
&& let Ok(val) = v
{
config.content_search.flush_interval_ms = val;
}
if let Ok(v) = env::var("OXICLOUD_CONTENT_INDEX_MAX_FILE_BYTES").map(|v| v.parse::<u64>())
&& let Ok(val) = v
{
config.content_search.max_extract_file_bytes = val;
}
if let Ok(v) = env::var("OXICLOUD_CONTENT_INDEX_MAX_TEXT_BYTES").map(|v| v.parse::<usize>())
&& let Ok(val) = v
{
config.content_search.max_text_bytes = val;
}
if let Ok(v) = env::var("OXICLOUD_EXPOSE_SYSTEM_USERS").map(|v| v.parse::<bool>())
&& let Ok(val) = v
{
+77 -1
View File
@@ -40,6 +40,8 @@ use crate::infrastructure::services::file_system_i18n_service::FileSystemI18nSer
use crate::infrastructure::services::nextcloud_chunked_upload_service::NextcloudChunkedUploadService;
use crate::infrastructure::services::path_service::PathService;
use crate::infrastructure::services::pg_acl_engine::PgAclEngine;
use crate::infrastructure::services::search_index::content_index_worker::ContentIndexWorker;
use crate::infrastructure::services::search_index::tantivy_content_index::TantivyContentIndex;
use crate::infrastructure::services::trash_cleanup_service::TrashCleanupService;
use crate::application::services::app_password_service::AppPasswordService;
@@ -440,6 +442,7 @@ impl AppServiceFactory {
trash_service: Option<Arc<TrashService>>,
authz: &Arc<PgAclEngine>,
storage_usage: &Arc<StorageUsageService>,
content_index: Option<Arc<TantivyContentIndex>>,
) -> ApplicationServices {
// Main services
let folder_service = Arc::new(FolderService::new(
@@ -503,10 +506,15 @@ impl AppServiceFactory {
let i18n_service = Arc::new(I18nApplicationService::new(repos.i18n_repository.clone()));
// Search service with cache
// Search service with cache. The optional content index widens the
// same `/api/search` endpoint to full-text content matches.
let content_index_port: Option<
Arc<dyn crate::application::ports::content_index_ports::ContentIndexPort>,
> = content_index.map(|idx| idx as _);
let search_service: Option<Arc<SearchService>> = Some(Arc::new(SearchService::new(
repos.file_read_repository.clone(),
repos.folder_repository.clone(),
content_index_port,
300, // Cache TTL in seconds (5 minutes)
1000, // Maximum cache entries
)));
@@ -711,6 +719,66 @@ impl AppServiceFactory {
tracing::info!("Tree-ETag flush service initialized");
}
/// Opens (or rebuilds) the embedded Tantivy content index. Returns the
/// index plus a reseed flag (true when the on-disk index was missing or
/// version-stale and must be repopulated from `storage.files`). Any
/// failure degrades to name-only search instead of failing startup.
fn create_content_index(&self) -> Option<(Arc<TantivyContentIndex>, bool)> {
if !self.config.content_search.enabled {
tracing::info!("Content search is disabled in configuration");
return None;
}
let dir = self
.config
.content_search
.index_dir
.clone()
.unwrap_or_else(|| self.storage_path.join(".search-index"));
match TantivyContentIndex::open_or_rebuild(&dir) {
Ok((index, needs_reseed)) => {
tracing::info!(
"Content index ready at {} ({} doc(s), reseed: {})",
dir.display(),
index.num_docs(),
needs_reseed
);
Some((Arc::new(index), needs_reseed))
}
Err(e) => {
tracing::error!("Content index unavailable — search will be name-only: {e}");
None
}
}
}
/// Starts the content-index pipeline on the maintenance pool. The
/// `storage.files` triggers enqueue unconditionally, so when the feature
/// is off (or the index failed to open) a discard-only janitor keeps the
/// dirty queue bounded instead.
fn start_content_index_job(
&self,
maintenance_pool: &Arc<PgPool>,
core: &CoreServices,
content_index: Option<(Arc<TantivyContentIndex>, bool)>,
) {
match content_index {
Some((index, needs_reseed)) => {
ContentIndexWorker::new(
maintenance_pool.clone(),
core.dedup_service.clone(),
index,
self.config.content_search.flush_interval_ms,
self.config.content_search.max_extract_file_bytes,
self.config.content_search.max_text_bytes,
)
.start(needs_reseed);
tracing::info!("Content-index worker initialized");
}
None => ContentIndexWorker::start_drain_only_janitor(maintenance_pool.clone()),
}
}
/// Builds the complete AppState using all factory services.
///
/// This is the main entry point that replaces all manual logic in `main.rs`.
@@ -759,6 +827,11 @@ impl AppServiceFactory {
// for the handler-side quota checks of the byte-upload paths).
let storage_usage = self.create_storage_usage_service(&repos, &pool, &maintenance_pool);
// 3d. Content index (embedded Tantivy) — opened before application
// services so SearchService can hold the query port; the feeding
// worker starts further down with the maintenance pool.
let content_index = self.create_content_index();
// 4. Application services (with trash + authz already wired)
let mut apps = self.create_application_services(
&core,
@@ -766,6 +839,7 @@ impl AppServiceFactory {
trash_service.clone(),
&authorization,
&storage_usage,
content_index.as_ref().map(|(idx, _)| idx.clone()),
);
// 5. Share service
@@ -810,6 +884,8 @@ impl AppServiceFactory {
self.start_tree_etag_flush_job(&maintenance_pool);
self.start_content_index_job(&maintenance_pool, &core, content_index);
// User-lifecycle dispatcher. Hook order is registration order;
// document dependencies inline if/when any arise. Today:
// 1. AuditLifecycleHook — fires first so the