Add embedded Tantivy full-text content search
/api/search now finds files by CONTENT as well as by name: BM25-ranked
matches over extracted text (PDF, Office OOXML/ODF, plain text/code)
with typo-tolerant fuzzy terms and search-as-you-type prefix matching,
served from an embedded Tantivy index at {storage}/.search-index.
Pipeline (all off the request path, mirroring tree-etag + thumbnails):
- statement triggers on storage.files append to a durable dirty queue
(storage.search_index_dirty) - every write surface (REST, WebDAV,
NextCloud, WOPI, trash) is covered, crash-safe by construction
- ContentIndexWorker drains the queue on the maintenance pool, extracts
text once per unique BLAKE3 blob (storage.blob_extracted_text cache:
N copies = 1 extraction, renames/moves = 0 re-extraction) and applies
batched single-writer Tantivy commits; queue rows are deleted only
after the commit succeeds (at-least-once, idempotent upserts)
- the index is a derived artifact: a version-marker mismatch wipes and
reseeds it from Postgres, which remains the single source of truth
SearchService merges content hits into the existing name search: hits
are hydrated through ONE SQL round-trip that re-applies user scope,
trash state and every active filter (a stale index id can never leak),
scored below name matches, and returned with a plain-text snippet and
a match_source field. Index failure or
OXICLOUD_ENABLE_CONTENT_SEARCH=false degrades to name-only search; a
discard-only janitor keeps the trigger-fed queue bounded while disabled.
The frontend renders the snippet under the file name in list view.
New dependencies: tantivy 0.26, zip 8.6 (deflate only), pdf-extract 0.10.
https://claude.ai/code/session_01Sc7F4xbo83YbFAQ4xEeDrX
This commit is contained in:
@@ -905,6 +905,44 @@ impl Default for FeaturesConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Content-search configuration (embedded Tantivy index over file names and
|
||||
/// extracted file content).
|
||||
///
|
||||
/// The index is a derived artifact fed by a background worker on the
|
||||
/// maintenance pool — none of these knobs affect request-path latency.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ContentSearchConfig {
|
||||
/// Master switch. When disabled, search falls back to name-only SQL and
|
||||
/// a janitor keeps the (always-installed) dirty queue empty.
|
||||
/// Env: `OXICLOUD_ENABLE_CONTENT_SEARCH`.
|
||||
pub enabled: bool,
|
||||
/// Index directory. Default: `{storage_path}/.search-index`.
|
||||
/// Env: `OXICLOUD_CONTENT_INDEX_DIR`.
|
||||
pub index_dir: Option<PathBuf>,
|
||||
/// Worker drain cadence in milliseconds — the upper bound on how long a
|
||||
/// new upload takes to become content-searchable. Default: 1500.
|
||||
/// Env: `OXICLOUD_CONTENT_INDEX_FLUSH_MS`.
|
||||
pub flush_interval_ms: u64,
|
||||
/// Files larger than this are indexed by NAME only (no text extraction).
|
||||
/// Default: 32 MiB. Env: `OXICLOUD_CONTENT_INDEX_MAX_FILE_BYTES`.
|
||||
pub max_extract_file_bytes: u64,
|
||||
/// Hard cap on extracted text per blob fed to the index. Default: 1 MiB.
|
||||
/// Env: `OXICLOUD_CONTENT_INDEX_MAX_TEXT_BYTES`.
|
||||
pub max_text_bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for ContentSearchConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
index_dir: None,
|
||||
flush_interval_ms: 1500,
|
||||
max_extract_file_bytes: 32 * 1024 * 1024,
|
||||
max_text_bytes: 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Global application configuration
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AppConfig {
|
||||
@@ -944,6 +982,8 @@ pub struct AppConfig {
|
||||
pub magic_link: MagicLinkConfig,
|
||||
/// I18n configuration (default locale for server-rendered surfaces)
|
||||
pub i18n: I18nConfig,
|
||||
/// Content-search configuration (embedded full-text index)
|
||||
pub content_search: ContentSearchConfig,
|
||||
}
|
||||
|
||||
/// Server-side i18n knobs.
|
||||
@@ -995,6 +1035,7 @@ impl Default for AppConfig {
|
||||
smtp: SmtpConfig::default(),
|
||||
magic_link: MagicLinkConfig::default(),
|
||||
i18n: I18nConfig::default(),
|
||||
content_search: ContentSearchConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1257,6 +1298,33 @@ impl AppConfig {
|
||||
config.features.enable_music = val;
|
||||
}
|
||||
|
||||
// Content search (embedded Tantivy index)
|
||||
if let Ok(v) = env::var("OXICLOUD_ENABLE_CONTENT_SEARCH").map(|v| v.parse::<bool>())
|
||||
&& let Ok(val) = v
|
||||
{
|
||||
config.content_search.enabled = val;
|
||||
}
|
||||
if let Ok(dir) = env::var("OXICLOUD_CONTENT_INDEX_DIR")
|
||||
&& !dir.trim().is_empty()
|
||||
{
|
||||
config.content_search.index_dir = Some(PathBuf::from(dir.trim()));
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_CONTENT_INDEX_FLUSH_MS").map(|v| v.parse::<u64>())
|
||||
&& let Ok(val) = v
|
||||
{
|
||||
config.content_search.flush_interval_ms = val;
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_CONTENT_INDEX_MAX_FILE_BYTES").map(|v| v.parse::<u64>())
|
||||
&& let Ok(val) = v
|
||||
{
|
||||
config.content_search.max_extract_file_bytes = val;
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_CONTENT_INDEX_MAX_TEXT_BYTES").map(|v| v.parse::<usize>())
|
||||
&& let Ok(val) = v
|
||||
{
|
||||
config.content_search.max_text_bytes = val;
|
||||
}
|
||||
|
||||
if let Ok(v) = env::var("OXICLOUD_EXPOSE_SYSTEM_USERS").map(|v| v.parse::<bool>())
|
||||
&& let Ok(val) = v
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user