Merge origin/main (Tantivy content search) into delta-sync branch
Both sides added a parameter to create_application_services and a setup step before it: this branch's storage-usage/quota service (for the instant-upload path) and main's Tantivy content index (for SearchService). The resolution keeps both — the signature takes both arguments and the build runs storage usage as step 3c and the content index as 3d. https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS
This commit is contained in:
@@ -898,6 +898,44 @@ impl Default for FeaturesConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Content-search configuration (embedded Tantivy index over file names and
|
||||
/// extracted file content).
|
||||
///
|
||||
/// The index is a derived artifact fed by a background worker on the
|
||||
/// maintenance pool — none of these knobs affect request-path latency.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ContentSearchConfig {
|
||||
/// Master switch. When disabled, search falls back to name-only SQL and
|
||||
/// a janitor keeps the (always-installed) dirty queue empty.
|
||||
/// Env: `OXICLOUD_ENABLE_CONTENT_SEARCH`.
|
||||
pub enabled: bool,
|
||||
/// Index directory. Default: `{storage_path}/.search-index`.
|
||||
/// Env: `OXICLOUD_CONTENT_INDEX_DIR`.
|
||||
pub index_dir: Option<PathBuf>,
|
||||
/// Worker drain cadence in milliseconds — the upper bound on how long a
|
||||
/// new upload takes to become content-searchable. Default: 1500.
|
||||
/// Env: `OXICLOUD_CONTENT_INDEX_FLUSH_MS`.
|
||||
pub flush_interval_ms: u64,
|
||||
/// Files larger than this are indexed by NAME only (no text extraction).
|
||||
/// Default: 32 MiB. Env: `OXICLOUD_CONTENT_INDEX_MAX_FILE_BYTES`.
|
||||
pub max_extract_file_bytes: u64,
|
||||
/// Hard cap on extracted text per blob fed to the index. Default: 1 MiB.
|
||||
/// Env: `OXICLOUD_CONTENT_INDEX_MAX_TEXT_BYTES`.
|
||||
pub max_text_bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for ContentSearchConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
index_dir: None,
|
||||
flush_interval_ms: 1500,
|
||||
max_extract_file_bytes: 32 * 1024 * 1024,
|
||||
max_text_bytes: 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Global application configuration
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AppConfig {
|
||||
@@ -937,6 +975,8 @@ pub struct AppConfig {
|
||||
pub magic_link: MagicLinkConfig,
|
||||
/// I18n configuration (default locale for server-rendered surfaces)
|
||||
pub i18n: I18nConfig,
|
||||
/// Content-search configuration (embedded full-text index)
|
||||
pub content_search: ContentSearchConfig,
|
||||
}
|
||||
|
||||
/// Server-side i18n knobs.
|
||||
@@ -988,6 +1028,7 @@ impl Default for AppConfig {
|
||||
smtp: SmtpConfig::default(),
|
||||
magic_link: MagicLinkConfig::default(),
|
||||
i18n: I18nConfig::default(),
|
||||
content_search: ContentSearchConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1250,6 +1291,33 @@ impl AppConfig {
|
||||
config.features.enable_music = val;
|
||||
}
|
||||
|
||||
// Content search (embedded Tantivy index)
|
||||
if let Ok(v) = env::var("OXICLOUD_ENABLE_CONTENT_SEARCH").map(|v| v.parse::<bool>())
|
||||
&& let Ok(val) = v
|
||||
{
|
||||
config.content_search.enabled = val;
|
||||
}
|
||||
if let Ok(dir) = env::var("OXICLOUD_CONTENT_INDEX_DIR")
|
||||
&& !dir.trim().is_empty()
|
||||
{
|
||||
config.content_search.index_dir = Some(PathBuf::from(dir.trim()));
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_CONTENT_INDEX_FLUSH_MS").map(|v| v.parse::<u64>())
|
||||
&& let Ok(val) = v
|
||||
{
|
||||
config.content_search.flush_interval_ms = val;
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_CONTENT_INDEX_MAX_FILE_BYTES").map(|v| v.parse::<u64>())
|
||||
&& let Ok(val) = v
|
||||
{
|
||||
config.content_search.max_extract_file_bytes = val;
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_CONTENT_INDEX_MAX_TEXT_BYTES").map(|v| v.parse::<usize>())
|
||||
&& let Ok(val) = v
|
||||
{
|
||||
config.content_search.max_text_bytes = val;
|
||||
}
|
||||
|
||||
if let Ok(v) = env::var("OXICLOUD_EXPOSE_SYSTEM_USERS").map(|v| v.parse::<bool>())
|
||||
&& let Ok(val) = v
|
||||
{
|
||||
|
||||
+77
-1
@@ -40,6 +40,8 @@ use crate::infrastructure::services::file_system_i18n_service::FileSystemI18nSer
|
||||
use crate::infrastructure::services::nextcloud_chunked_upload_service::NextcloudChunkedUploadService;
|
||||
use crate::infrastructure::services::path_service::PathService;
|
||||
use crate::infrastructure::services::pg_acl_engine::PgAclEngine;
|
||||
use crate::infrastructure::services::search_index::content_index_worker::ContentIndexWorker;
|
||||
use crate::infrastructure::services::search_index::tantivy_content_index::TantivyContentIndex;
|
||||
use crate::infrastructure::services::trash_cleanup_service::TrashCleanupService;
|
||||
|
||||
use crate::application::services::app_password_service::AppPasswordService;
|
||||
@@ -440,6 +442,7 @@ impl AppServiceFactory {
|
||||
trash_service: Option<Arc<TrashService>>,
|
||||
authz: &Arc<PgAclEngine>,
|
||||
storage_usage: &Arc<StorageUsageService>,
|
||||
content_index: Option<Arc<TantivyContentIndex>>,
|
||||
) -> ApplicationServices {
|
||||
// Main services
|
||||
let folder_service = Arc::new(FolderService::new(
|
||||
@@ -503,10 +506,15 @@ impl AppServiceFactory {
|
||||
|
||||
let i18n_service = Arc::new(I18nApplicationService::new(repos.i18n_repository.clone()));
|
||||
|
||||
// Search service with cache
|
||||
// Search service with cache. The optional content index widens the
|
||||
// same `/api/search` endpoint to full-text content matches.
|
||||
let content_index_port: Option<
|
||||
Arc<dyn crate::application::ports::content_index_ports::ContentIndexPort>,
|
||||
> = content_index.map(|idx| idx as _);
|
||||
let search_service: Option<Arc<SearchService>> = Some(Arc::new(SearchService::new(
|
||||
repos.file_read_repository.clone(),
|
||||
repos.folder_repository.clone(),
|
||||
content_index_port,
|
||||
300, // Cache TTL in seconds (5 minutes)
|
||||
1000, // Maximum cache entries
|
||||
)));
|
||||
@@ -711,6 +719,66 @@ impl AppServiceFactory {
|
||||
tracing::info!("Tree-ETag flush service initialized");
|
||||
}
|
||||
|
||||
/// Opens (or rebuilds) the embedded Tantivy content index. Returns the
|
||||
/// index plus a reseed flag (true when the on-disk index was missing or
|
||||
/// version-stale and must be repopulated from `storage.files`). Any
|
||||
/// failure degrades to name-only search instead of failing startup.
|
||||
fn create_content_index(&self) -> Option<(Arc<TantivyContentIndex>, bool)> {
|
||||
if !self.config.content_search.enabled {
|
||||
tracing::info!("Content search is disabled in configuration");
|
||||
return None;
|
||||
}
|
||||
let dir = self
|
||||
.config
|
||||
.content_search
|
||||
.index_dir
|
||||
.clone()
|
||||
.unwrap_or_else(|| self.storage_path.join(".search-index"));
|
||||
|
||||
match TantivyContentIndex::open_or_rebuild(&dir) {
|
||||
Ok((index, needs_reseed)) => {
|
||||
tracing::info!(
|
||||
"Content index ready at {} ({} doc(s), reseed: {})",
|
||||
dir.display(),
|
||||
index.num_docs(),
|
||||
needs_reseed
|
||||
);
|
||||
Some((Arc::new(index), needs_reseed))
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Content index unavailable — search will be name-only: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts the content-index pipeline on the maintenance pool. The
|
||||
/// `storage.files` triggers enqueue unconditionally, so when the feature
|
||||
/// is off (or the index failed to open) a discard-only janitor keeps the
|
||||
/// dirty queue bounded instead.
|
||||
fn start_content_index_job(
|
||||
&self,
|
||||
maintenance_pool: &Arc<PgPool>,
|
||||
core: &CoreServices,
|
||||
content_index: Option<(Arc<TantivyContentIndex>, bool)>,
|
||||
) {
|
||||
match content_index {
|
||||
Some((index, needs_reseed)) => {
|
||||
ContentIndexWorker::new(
|
||||
maintenance_pool.clone(),
|
||||
core.dedup_service.clone(),
|
||||
index,
|
||||
self.config.content_search.flush_interval_ms,
|
||||
self.config.content_search.max_extract_file_bytes,
|
||||
self.config.content_search.max_text_bytes,
|
||||
)
|
||||
.start(needs_reseed);
|
||||
tracing::info!("Content-index worker initialized");
|
||||
}
|
||||
None => ContentIndexWorker::start_drain_only_janitor(maintenance_pool.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the complete AppState using all factory services.
|
||||
///
|
||||
/// This is the main entry point that replaces all manual logic in `main.rs`.
|
||||
@@ -759,6 +827,11 @@ impl AppServiceFactory {
|
||||
// for the handler-side quota checks of the byte-upload paths).
|
||||
let storage_usage = self.create_storage_usage_service(&repos, &pool, &maintenance_pool);
|
||||
|
||||
// 3d. Content index (embedded Tantivy) — opened before application
|
||||
// services so SearchService can hold the query port; the feeding
|
||||
// worker starts further down with the maintenance pool.
|
||||
let content_index = self.create_content_index();
|
||||
|
||||
// 4. Application services (with trash + authz already wired)
|
||||
let mut apps = self.create_application_services(
|
||||
&core,
|
||||
@@ -766,6 +839,7 @@ impl AppServiceFactory {
|
||||
trash_service.clone(),
|
||||
&authorization,
|
||||
&storage_usage,
|
||||
content_index.as_ref().map(|(idx, _)| idx.clone()),
|
||||
);
|
||||
|
||||
// 5. Share service
|
||||
@@ -810,6 +884,8 @@ impl AppServiceFactory {
|
||||
|
||||
self.start_tree_etag_flush_job(&maintenance_pool);
|
||||
|
||||
self.start_content_index_job(&maintenance_pool, &core, content_index);
|
||||
|
||||
// User-lifecycle dispatcher. Hook order is registration order;
|
||||
// document dependencies inline if/when any arise. Today:
|
||||
// 1. AuditLifecycleHook — fires first so the
|
||||
|
||||
Reference in New Issue
Block a user