cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

perf: Phase 4+5 optimizations — uploads 10x, downloads 2x, concurrent 2x. moka cache, 512KB buffers, remove sync_all, hash-on-write, preloaded queries, bench.sh v3, gitignore storage/. 500MB upload 12.6s->1.3s (392MB/s). RSS 69-113MB, 0 swap.

Browse Source
This commit is contained in:
Dionisio
2026-02-15 17:53:25 +01:00
parent fac0b5e77b
commit 1ed20f425f
57 changed files with 1700 additions and 1184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3
View File
@@ -56,6 +56,9 @@ npm-debug.log
*.log
logs/
# Storage data (user files, blobs — never commit)
storage/
# Temporary files
*.tmp
*.bak
Cargo.lock
Generated
+72
View File
@@ -88,6 +88,17 @@ dependencies = [
"tokio",
]
[[package]]
name = "async-lock"
version = "3.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311"
dependencies = [
"event-listener",
"event-listener-strategy",
"pin-project-lite",
]
[[package]]
name = "async-stream"
version = "0.3.6"
@@ -422,6 +433,24 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.12"
@@ -673,6 +702,16 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "event-listener-strategy"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93"
dependencies = [
"event-listener",
"pin-project-lite",
]
[[package]]
name = "fastrand"
version = "2.3.0"
@@ -1584,6 +1623,26 @@ dependencies = [
"syn",
]
[[package]]
name = "moka"
version = "0.12.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ac832c50ced444ef6be0767a008b02c106a909ba79d1d830501e94b96f6b7e"
dependencies = [
"async-lock",
"crossbeam-channel",
"crossbeam-epoch",
"crossbeam-utils",
"equivalent",
"event-listener",
"futures-util",
"parking_lot",
"portable-atomic",
"smallvec",
"tagptr",
"uuid",
]
[[package]]
name = "moxcms"
version = "0.7.11"
@@ -1716,6 +1775,7 @@ dependencies = [
"memmap2",
"mime_guess",
"mockall",
"moka",
"quick-xml",
"rand_core 0.6.4",
"reqwest",
@@ -1887,6 +1947,12 @@ dependencies = [
"miniz_oxide",
]
[[package]]
name = "portable-atomic"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
[[package]]
name = "potential_utf"
version = "0.1.4"
@@ -2769,6 +2835,12 @@ dependencies = [
"syn",
]
[[package]]
name = "tagptr"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]]
name = "tempfile"
version = "3.25.0"
Cargo.toml
+1
View File
@@ -38,6 +38,7 @@ hyper = { version = "1.8.1", features = ["full"] }
quick-xml = "0.39.0"
dotenvy = "0.15.7"
lru = "0.16.3"
moka = { version = "0.12", features = ["future"] }
memmap2 = "0.9"
http-range-header = "0.4"
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "gif", "webp"] }
bench.sh
Executable
+419
View File
@@ -0,0 +1,419 @@
#!/usr/bin/env bash
# ============================================================================
# OxiCloud Performance Benchmark v3
# Detailed RAM (RSS/VmPeak/VmSwap) + CPU monitoring + huge file downloads
#
# - Uploads: 1KB..500MB via API (above ~1GB the upload handler OOMs —
# that's a known issue in the multipart handler which buffers chunks).
# - Downloads: 1KB..10GB. For files >500MB we inject blobs directly into
# the blob store + DB so the streaming download path is tested without
# going through the buffering upload handler.
# ============================================================================
set -uo pipefail
BASE_URL="${BASE_URL:-http://127.0.0.1:8086}"
USERNAME="${BENCH_USER:-benchuser}"
PASSWORD="${BENCH_PASS:-BenchPass123!}"
FOLDER_ID="${BENCH_FOLDER:-0e72efc0-0d1c-45a1-b434-52336643b3f7}"
USER_ID="${BENCH_USER_ID:-c410b103-7b86-4ac2-9eb4-3804351547be}"
WORKDIR="/tmp/oxibench"
RESULTS="$WORKDIR/results_v3.csv"
STORAGE="./storage"
MONITOR_INTERVAL=0.2 # 200ms sample rate
# ── Colours ──────────────────────────────────────────────────────────────────
RED='\033[0;31m'; GRN='\033[0;32m'; YEL='\033[1;33m'
CYA='\033[0;36m'; BLD='\033[1m'; DIM='\033[2m'; NC='\033[0m'
header() { printf "\n${BLD}${CYA}═══════════════════════════════════════════${NC}\n"; \
printf "${BLD}${CYA} %s${NC}\n" "$1"; \
printf "${BLD}${CYA}═══════════════════════════════════════════${NC}\n"; }
ok() { printf "${GRN}✓${NC} %s\n" "$1"; }
warn() { printf "${YEL}⚠${NC} %s\n" "$1"; }
fail() { printf "${RED}✗${NC} %s\n" "$1"; }
# ── Server PID ───────────────────────────────────────────────────────────────
SERVER_PID=""
find_server_pid() {
SERVER_PID=$(pgrep -f "target/release/oxicloud" 2>/dev/null | head -1 || \
pgrep -f "target/debug/oxicloud" 2>/dev/null | head -1 || echo "")
[[ -z "$SERVER_PID" ]] && { fail "Server PID not found"; exit 1; }
}
# ── Memory (KB) ─────────────────────────────────────────────────────────────
get_rss_kb() { awk '/^VmRSS:/ {print $2}' /proc/$SERVER_PID/status 2>/dev/null || echo 0; }
get_peak_kb() { awk '/^VmPeak:/ {print $2}' /proc/$SERVER_PID/status 2>/dev/null || echo 0; }
get_swap_kb() { awk '/^VmSwap:/ {print $2}' /proc/$SERVER_PID/status 2>/dev/null || echo 0; }
get_vsize_kb() { awk '/^VmSize:/ {print $2}' /proc/$SERVER_PID/status 2>/dev/null || echo 0; }
# ── CPU jiffies ──────────────────────────────────────────────────────────────
get_cpu_jiffies() { awk '{print $14+$15}' /proc/$SERVER_PID/stat 2>/dev/null || echo 0; }
# ── Background monitor ──────────────────────────────────────────────────────
MONITOR_PID=""
MONITOR_RESULT="$WORKDIR/_mon"
MON_PEAK_RSS=0; MON_PEAK_VM=0; MON_PEAK_SWAP=0; MON_CPU_PCT="0.0"
start_monitor() {
rm -f "$MONITOR_RESULT"
(
peak_rss=0; peak_vm=0; peak_swap=0
cpu0=$(get_cpu_jiffies); t0=$(date +%s%N)
while true; do
r=$(get_rss_kb); v=$(get_vsize_kb); s=$(get_swap_kb)
(( r > peak_rss )) && peak_rss=$r
(( v > peak_vm )) && peak_vm=$v
(( s > peak_swap )) && peak_swap=$s
sleep $MONITOR_INTERVAL 2>/dev/null || break
done
cpu1=$(get_cpu_jiffies); t1=$(date +%s%N)
dt=$(( cpu1 - cpu0 )); wall=$(( (t1 - t0) / 1000000 ))
cpup=0
(( wall > 0 )) && cpup=$(echo "scale=1; $dt * 10 * 100 / $wall" | bc 2>/dev/null || echo 0)
echo "$peak_rss $peak_vm $peak_swap $cpup" > "$MONITOR_RESULT"
) &
MONITOR_PID=$!
}
stop_monitor() {
[[ -n "$MONITOR_PID" ]] && { kill $MONITOR_PID 2>/dev/null; wait $MONITOR_PID 2>/dev/null || true; MONITOR_PID=""; }
if [[ -f "$MONITOR_RESULT" ]]; then
read -r MON_PEAK_RSS MON_PEAK_VM MON_PEAK_SWAP MON_CPU_PCT < "$MONITOR_RESULT"
else
MON_PEAK_RSS=0; MON_PEAK_VM=0; MON_PEAK_SWAP=0; MON_CPU_PCT="0.0"
fi
}
# ── Auth ─────────────────────────────────────────────────────────────────────
TOKEN=""
login() {
TOKEN=$(curl -sf -X POST "$BASE_URL/api/auth/login" \
-H "Content-Type: application/json" \
-d "{\"username\":\"$USERNAME\",\"password\":\"$PASSWORD\"}" \
| python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])" 2>/dev/null || echo "")
[[ -z "$TOKEN" ]] && { fail "JWT login failed"; exit 1; }
}
# ── File gen ─────────────────────────────────────────────────────────────────
gen_file() {
local path="$1" bytes="$2"
local bs=1048576
local count=$(( bytes / bs )) rem=$(( bytes % bs ))
(( count > 0 )) && dd if=/dev/urandom of="$path" bs=$bs count=$count 2>/dev/null
(( rem > 0 )) && dd if=/dev/urandom bs=$rem count=1 >> "$path" 2>/dev/null
}
human() {
local b=$1
if (( b >= 1073741824 )); then printf "%.2f GB" "$(echo "$b/1073741824" | bc -l)"
elif (( b >= 1048576 )); then printf "%.1f MB" "$(echo "$b/1048576" | bc -l)"
elif (( b >= 1024 )); then printf "%.1f KB" "$(echo "$b/1024" | bc -l)"
else printf "%d B" "$b"; fi
}
throughput() {
local b=$1 ms=$2
(( ms <= 0 )) && { echo "---"; return; }
human $(echo "$b*1000/$ms" | bc 2>/dev/null || echo 0)
}
mb() { echo $(( $1 / 1024 )); } # KB -> MB
# ── Inject large blob (bypass upload handler) ──────────────────────────────
# Creates a random file as a blob, registers it in DB, returns file_id.
# Usage: inject_blob <label> <size_bytes>
inject_blob() {
local label="$1" size="$2"
local hash fpath blob_dir blob_path file_id
hash=$(printf "%s_%s" "$label" "$(date +%s%N)" | sha256sum | awk '{print $1}')
blob_dir="$STORAGE/.blobs/${hash:0:2}"
blob_path="$blob_dir/${hash}.blob"
mkdir -p "$blob_dir"
printf " Generating blob %s (%s) ..." "$label" "$(human $size)" >&2
gen_file "$blob_path" "$size"
printf " done\n" >&2
# Register blob in dedup index
docker exec -i $(docker ps -qf "name=postgres") psql -U postgres -d oxicloud -q <<SQL >/dev/null 2>&1
INSERT INTO storage.blobs (hash, size, content_type, ref_count)
VALUES ('$hash', $size, 'application/octet-stream', 1)
ON CONFLICT DO NOTHING;
SQL
# Register file metadata
file_id=$(docker exec -i $(docker ps -qf "name=postgres") psql -U postgres -d oxicloud -t -A <<SQL 2>/dev/null
INSERT INTO storage.files (name, folder_id, user_id, blob_hash, size, mime_type)
VALUES ('bench_${label}.bin', '$FOLDER_ID'::uuid, '$USER_ID', '$hash', $size, 'application/octet-stream')
RETURNING id::text;
SQL
)
file_id=$(echo "$file_id" | grep -oE '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' | head -1)
echo "$file_id"
}
# ── Setup ────────────────────────────────────────────────────────────────────
mkdir -p "$WORKDIR"
rm -f "$RESULTS"
echo "test,size_bytes,direction,latency_ms,rss_before_mb,rss_peak_mb,rss_after_mb,swap_peak_mb,cpu_pct,throughput_MBs,http_code" > "$RESULTS"
find_server_pid
header "OxiCloud Benchmark v3 — RAM + CPU + Huge Files"
printf " %-14s %s\n" "Server" "$BASE_URL"
printf " %-14s %s\n" "PID" "$SERVER_PID"
printf " %-14s %d MB\n" "RSS now" $(mb $(get_rss_kb))
printf " %-14s %d MB\n" "VmPeak" $(mb $(get_peak_kb))
printf " %-14s %d MB\n" "Swap" $(mb $(get_swap_kb))
printf " %-14s %s\n" "Cores" "$(nproc)"
printf " %-14s %s\n" "Disk free" "$(df -h $WORKDIR | tail -1 | awk '{print $4}')"
login; ok "Authenticated"
echo
# ── Upload sizes (safe for the buffering handler) ───────────────────────────
UPLOAD_SIZES=(
"1KB:1024"
"64KB:65536"
"256KB:262144"
"1MB:1048576"
"10MB:10485760"
"50MB:52428800"
"100MB:104857600"
"250MB:262144000"
"500MB:524288000"
)
# Additional sizes for download-only (injected directly as blobs)
DOWNLOAD_ONLY_SIZES=(
"1GB:1073741824"
"2GB:2147483648"
"5GB:5368709120"
"10GB:10737418240"
)
# ── Generate upload test files ──────────────────────────────────────────────
header "Generating Upload Test Files"
for e in "${UPLOAD_SIZES[@]}"; do
l="${e%%:*}"; b="${e##*:}"
f="$WORKDIR/test_${l}.bin"
if [[ -f "$f" ]] && [[ $(stat -c%s "$f" 2>/dev/null || echo 0) -eq "$b" ]]; then
printf " %s exists\n" "$l"
else
printf " %s (%s) ..." "$l" "$(human $b)"
gen_file "$f" "$b"
printf " done\n"
fi
done
# ============================================================================
# UPLOADS (via API multipart)
# ============================================================================
header "Upload Benchmarks (via API)"
printf "${BLD}%-8s │ %8s %10s │ %7s %7s %7s %6s │ %5s │ %4s${NC}\n" \
"Size" "Latency" "Thruput" "RSSbef" "RSSpk" "RSSaft" "Swap" "CPU%" "HTTP"
printf "%-8s │ %8s %10s │ %7s %7s %7s %6s │ %5s │ %4s\n" \
"------" "-------" "--------" "------" "-----" "------" "----" "----" "----"
declare -A FILE_IDS
for e in "${UPLOAD_SIZES[@]}"; do
l="${e%%:*}"; b="${e##*:}"; f="$WORKDIR/test_${l}.bin"
login >/dev/null 2>&1
rss0=$(get_rss_kb)
start_monitor
t0=$(date +%s%N)
resp=$(curl -s -w '\n%{http_code}' --max-time 3600 \
-X POST "$BASE_URL/api/files/upload" \
-H "Authorization: Bearer $TOKEN" \
-F "folder_id=$FOLDER_ID" \
-F "file=@$f" 2>&1)
t1=$(date +%s%N)
stop_monitor
code=$(echo "$resp" | tail -1)
body=$(echo "$resp" | head -n -1)
ms=$(( (t1 - t0) / 1000000 ))
rss1=$(get_rss_kb)
tp=$(throughput "$b" "$ms")
fid=$(echo "$body" | python3 -c "import sys,json;print(json.load(sys.stdin)['id'])" 2>/dev/null || echo "")
FILE_IDS[$l]="$fid"
printf "%-8s │ %6dms %8s/s │ %5dMB %5dMB %5dMB %4dMB │ %5s │ %4s\n" \
"$l" "$ms" "$tp" \
$(mb $rss0) $(mb $MON_PEAK_RSS) $(mb $rss1) $(mb $MON_PEAK_SWAP) \
"$MON_CPU_PCT" "$code"
tpm=$(echo "scale=2;$b*1000/($ms+1)/1048576" | bc 2>/dev/null || echo 0)
echo "upload_$l,$b,upload,$ms,$(mb $rss0),$(mb $MON_PEAK_RSS),$(mb $rss1),$(mb $MON_PEAK_SWAP),$MON_CPU_PCT,$tpm,$code" >> "$RESULTS"
sleep 1
done
# ============================================================================
# INJECT LARGE BLOBS (for download-only tests)
# ============================================================================
header "Injecting Large Blobs (bypass upload handler)"
for e in "${DOWNLOAD_ONLY_SIZES[@]}"; do
l="${e%%:*}"; b="${e##*:}"
fid=$(inject_blob "$l" "$b")
FILE_IDS[$l]="$fid"
ok "Registered $l → $fid"
done
# ============================================================================
# DOWNLOADS (all sizes including injected blobs)
# ============================================================================
ALL_SIZES=("${UPLOAD_SIZES[@]}" "${DOWNLOAD_ONLY_SIZES[@]}")
header "Download Benchmarks"
printf "${BLD}%-8s │ %8s %10s │ %7s %7s %7s %6s │ %5s │ %4s${NC}\n" \
"Size" "Latency" "Thruput" "RSSbef" "RSSpk" "RSSaft" "Swap" "CPU%" "HTTP"
printf "%-8s │ %8s %10s │ %7s %7s %7s %6s │ %5s │ %4s\n" \
"------" "-------" "--------" "------" "-----" "------" "----" "----" "----"
for e in "${ALL_SIZES[@]}"; do
l="${e%%:*}"; b="${e##*:}"
fid="${FILE_IDS[$l]:-}"
[[ -z "$fid" ]] && { warn "$l skipped (no ID)"; continue; }
login >/dev/null 2>&1
rss0=$(get_rss_kb)
start_monitor
t0=$(date +%s%N)
code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 7200 \
"$BASE_URL/api/files/$fid" -H "Authorization: Bearer $TOKEN" 2>&1)
t1=$(date +%s%N)
stop_monitor
ms=$(( (t1 - t0) / 1000000 ))
rss1=$(get_rss_kb)
tp=$(throughput "$b" "$ms")
printf "%-8s │ %6dms %8s/s │ %5dMB %5dMB %5dMB %4dMB │ %5s │ %4s\n" \
"$l" "$ms" "$tp" \
$(mb $rss0) $(mb $MON_PEAK_RSS) $(mb $rss1) $(mb $MON_PEAK_SWAP) \
"$MON_CPU_PCT" "$code"
tpm=$(echo "scale=2;$b*1000/($ms+1)/1048576" | bc 2>/dev/null || echo 0)
echo "download_$l,$b,download,$ms,$(mb $rss0),$(mb $MON_PEAK_RSS),$(mb $rss1),$(mb $MON_PEAK_SWAP),$MON_CPU_PCT,$tpm,$code" >> "$RESULTS"
sleep 1
done
# ============================================================================
# RANGE REQUESTS (first 64 KB — constant memory regardless of file size)
# ============================================================================
header "Range Requests (64 KB slice from each file)"
printf "${BLD}%-8s │ %8s │ %7s %7s │ %4s${NC}\n" "Size" "Latency" "RSSpk" "RSSdelta" "HTTP"
printf "%-8s │ %8s │ %7s %7s │ %4s\n" "------" "-------" "-----" "-------" "----"
for e in "${ALL_SIZES[@]}"; do
l="${e%%:*}"; b="${e##*:}"
fid="${FILE_IDS[$l]:-}"
[[ -z "$fid" ]] && continue
login >/dev/null 2>&1
rss0=$(get_rss_kb)
start_monitor
t0=$(date +%s%N)
code=$(curl -s -o /dev/null -w '%{http_code}' \
"$BASE_URL/api/files/$fid" \
-H "Authorization: Bearer $TOKEN" \
-H "Range: bytes=0-65535" 2>&1)
t1=$(date +%s%N)
stop_monitor
ms=$(( (t1 - t0) / 1000000 ))
rss1=$(get_rss_kb)
d=$(( rss1 - rss0 ))
printf "%-8s │ %6dms │ %5dMB %+5dKB │ %4s\n" \
"$l" "$ms" $(mb $MON_PEAK_RSS) "$d" "$code"
echo "range_$l,$b,range,$ms,$(mb $rss0),$(mb $MON_PEAK_RSS),$(mb $rss1),$(mb $MON_PEAK_SWAP),$MON_CPU_PCT,0,$code" >> "$RESULTS"
done
# ============================================================================
# CONCURRENT DOWNLOADS
# ============================================================================
header "Concurrent Downloads"
run_concurrent() {
local lbl="$1" fid="$2" n="$3" each="$4"
local total=$(( each * n ))
login >/dev/null 2>&1
rss0=$(get_rss_kb)
start_monitor
t0=$(date +%s%N)
local pids=()
for i in $(seq 1 $n); do
curl -s -o /dev/null --max-time 7200 "$BASE_URL/api/files/$fid" \
-H "Authorization: Bearer $TOKEN" &
pids+=($!)
done
wait "${pids[@]}"
t1=$(date +%s%N)
stop_monitor
ms=$(( (t1 - t0) / 1000000 ))
rss1=$(get_rss_kb)
tp=$(throughput $total $ms)
printf " ${BLD}%dx %s${NC} %dms %s/s RSS:%dMB→peak %dMB→%dMB Swap:%dMB CPU:%s%%\n" \
"$n" "$lbl" "$ms" "$tp" \
$(mb $rss0) $(mb $MON_PEAK_RSS) $(mb $rss1) $(mb $MON_PEAK_SWAP) "$MON_CPU_PCT"
echo "concurrent_${n}x${lbl},$total,concurrent,$ms,$(mb $rss0),$(mb $MON_PEAK_RSS),$(mb $rss1),$(mb $MON_PEAK_SWAP),$MON_CPU_PCT,0,200" >> "$RESULTS"
}
fid="${FILE_IDS[10MB]:-}"; [[ -n "$fid" ]] && run_concurrent "10MB" "$fid" 5 10485760
fid="${FILE_IDS[100MB]:-}"; [[ -n "$fid" ]] && run_concurrent "100MB" "$fid" 3 104857600
fid="${FILE_IDS[1GB]:-}"; [[ -n "$fid" ]] && run_concurrent "1GB" "$fid" 2 1073741824
fid="${FILE_IDS[5GB]:-}"; [[ -n "$fid" ]] && run_concurrent "5GB" "$fid" 2 5368709120
# ============================================================================
# MEMORY TRACE during 5 GB download
# ============================================================================
header "Memory Trace: RSS during 5 GB download"
fid="${FILE_IDS[5GB]:-}"
if [[ -n "$fid" ]]; then
login >/dev/null 2>&1
TRACE="$WORKDIR/mem_trace_5gb.csv"
echo "elapsed_ms,rss_mb,vmsize_mb,swap_mb" > "$TRACE"
t0=$(date +%s%N)
(
while true; do
now=$(date +%s%N)
el=$(( (now - t0) / 1000000 ))
echo "$el,$(mb $(get_rss_kb)),$(mb $(get_vsize_kb)),$(mb $(get_swap_kb))" >> "$TRACE"
sleep 0.2
done
) &
SAMPLER=$!
curl -s -o /dev/null --max-time 7200 "$BASE_URL/api/files/$fid" -H "Authorization: Bearer $TOKEN"
kill $SAMPLER 2>/dev/null; wait $SAMPLER 2>/dev/null || true
mx=$(awk -F, 'NR>1{if($2>m)m=$2}END{print m+0}' "$TRACE")
mn=$(awk -F, 'NR>1{if(!m||$2<m)m=$2}END{print m+0}' "$TRACE")
samp=$(( $(wc -l < "$TRACE") - 1 ))
printf " Samples: %d (every 200ms)\n" "$samp"
printf " RSS min: %d MB\n" "$mn"
printf " RSS max: %d MB\n" "$mx"
printf " RSS delta: %d MB\n" $(( mx - mn ))
printf " Trace file: %s\n" "$TRACE"
else
warn "5GB file not available"
fi
# ============================================================================
# FINAL STATE
# ============================================================================
header "Final Server State"
printf " RSS: %5d MB\n" $(mb $(get_rss_kb))
printf " VmPeak: %5d MB\n" $(mb $(get_peak_kb))
printf " VmSize: %5d MB\n" $(mb $(get_vsize_kb))
printf " VmSwap: %5d MB\n" $(mb $(get_swap_kb))
echo
printf "${BLD}CSV Results:${NC}\n"
column -t -s',' "$RESULTS" 2>/dev/null || cat "$RESULTS"
echo
ok "Benchmark v3 complete — $RESULTS"
docker-compose.yml
+2 -2
View File
@@ -6,8 +6,8 @@ services:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: oxicloud
# ports:
# - "5432:5432"
ports:
- "5432:5432"
networks:
- oxicloud
volumes:
src/application/adapters/caldav_adapter.rs
+2 -6
View File
@@ -78,8 +78,7 @@ impl CalDavAdapter {
s if s == "filter" || s.ends_with(":filter") => in_filter = true,
s if s == "time-range" || s.ends_with(":time-range") => {
// Parse time-range attributes
for attr in e.attributes() {
if let Ok(attr) = attr {
for attr in e.attributes().flatten() {
let attr_name =
std::str::from_utf8(attr.key.as_ref()).unwrap_or("");
let attr_value = attr.unescape_value().unwrap_or_default();
@@ -96,7 +95,6 @@ impl CalDavAdapter {
}
}
}
}
s if s == "sync-token" || s.ends_with(":sync-token") => {
// We'll capture the text in the Text event
}
@@ -151,8 +149,7 @@ impl CalDavAdapter {
props.push(QualifiedName::new(namespace, prop_name));
} else if name_str == "time-range" || name_str.ends_with(":time-range") {
// Parse time-range attributes
for attr in e.attributes() {
if let Ok(attr) = attr {
for attr in e.attributes().flatten() {
let attr_name =
std::str::from_utf8(attr.key.as_ref()).unwrap_or("");
let attr_value = attr.unescape_value().unwrap_or_default();
@@ -170,7 +167,6 @@ impl CalDavAdapter {
}
}
}
}
Ok(Event::Eof) => break,
Err(e) => return Err(WebDavError::XmlError(e)),
_ => (),
src/application/adapters/webdav_adapter.rs
+6 -3
View File
@@ -61,11 +61,14 @@ impl QualifiedName {
}
}
pub fn to_string(&self) -> String {
}
impl std::fmt::Display for QualifiedName {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.namespace.is_empty() {
self.name.clone()
write!(f, "{}", self.name)
} else {
format!("{{{}}}{}", self.namespace, self.name)
write!(f, "{{{}}}{}", self.namespace, self.name)
}
}
}
src/application/dtos/pagination.rs
+1 -5
View File
@@ -73,11 +73,7 @@ impl PaginationRequestDto {
}
// Ensure the page size is between 10 and 500
if page_size < 10 {
page_size = 10;
} else if page_size > 500 {
page_size = 500;
}
page_size = page_size.clamp(10, 500);
Self { page, page_size }
}
src/application/ports/chunked_upload_ports.rs
+4 -2
View File
@@ -84,11 +84,13 @@ pub trait ChunkedUploadPort: Send + Sync + 'static {
/// Assemble all chunks into the final file.
///
/// Returns `(assembled_file_path, filename, folder_id, content_type, total_size)`.
/// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, sha256_hash)`.
/// The hash is computed during assembly (hash-on-write), eliminating a
/// second sequential read of the assembled file.
async fn complete_upload(
&self,
upload_id: &str,
) -> Result<(PathBuf, String, Option<String>, String, u64), DomainError>;
) -> Result<(PathBuf, String, Option<String>, String, u64, String), DomainError>;
/// Finalize upload: clean up the session and temporary files.
async fn finalize_upload(&self, upload_id: &str) -> Result<(), DomainError>;
src/application/ports/dedup_ports.rs
+30
View File
@@ -7,8 +7,10 @@
use crate::common::errors::DomainError;
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use serde::Serialize;
use std::path::{Path, PathBuf};
use std::pin::Pin;
/// Metadata of a stored blob in the dedup system.
#[derive(Debug, Clone, Serialize)]
@@ -103,10 +105,15 @@ pub trait DedupPort: Send + Sync + 'static {
) -> Result<DedupResultDto, DomainError>;
/// Store content with deduplication (streaming from file).
///
/// If `pre_computed_hash` is provided (e.g. hash-on-write from the handler),
/// the file will NOT be re-read to calculate the hash — saving one full
/// sequential read of the file.
async fn store_from_file(
&self,
source_path: &Path,
content_type: Option<String>,
pre_computed_hash: Option<String>,
) -> Result<DedupResultDto, DomainError>;
/// Check if a blob with the given hash exists.
@@ -121,6 +128,29 @@ pub trait DedupPort: Send + Sync + 'static {
/// Read blob content as `Bytes`.
async fn read_blob_bytes(&self, hash: &str) -> Result<Bytes, DomainError>;
/// Stream blob content in chunks (64 KB default) — constant memory usage.
///
/// Unlike `read_blob()`, this never loads the entire file into RAM.
async fn read_blob_stream(
&self,
hash: &str,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>;
/// Stream a byte range of a blob — only reads the requested portion.
///
/// Uses seek + take so a 1 MB range on a 1 GB file only reads 1 MB from disk.
async fn read_blob_range_stream(
&self,
hash: &str,
start: u64,
end: Option<u64>,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>;
/// Get the size of a blob without reading its content.
///
/// Used by HEAD requests to return Content-Length without loading the file.
async fn blob_size(&self, hash: &str) -> Result<u64, DomainError>;
/// Add a reference to a blob (increment ref_count).
async fn add_reference(&self, hash: &str) -> Result<(), DomainError>;
src/application/ports/file_ports.rs
+47 -20
View File
@@ -1,6 +1,7 @@
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use std::path::Path;
use std::pin::Pin;
use std::sync::Arc;
@@ -11,21 +12,33 @@ use crate::common::errors::DomainError;
// Upload port
// ─────────────────────────────────────────────────────
/// Strategy chosen by the upload service based on file size.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UploadStrategy {
/// Instant (<256KB): write-behind cache, ~0ms latency
WriteBehind,
/// Buffered (256KB–1MB): full bytes in memory then write
Buffered,
/// Streaming (≥1MB): pipe chunks directly to disk
Streaming,
}
/// Primary port for file upload operations
/// Primary port for file upload operations.
///
/// All upload paths converge on streaming-to-disk:
/// - Normal uploads: handler spools multipart to temp file → `upload_file_streaming`
/// - WebDAV PUT: small in-memory buffer → `upload_file`
/// - Chunked uploads: chunks already on disk → `upload_file_from_path`
#[async_trait]
pub trait FileUploadUseCase: Send + Sync + 'static {
/// Uploads a new file from bytes
/// Upload from a temp file already on disk (true streaming, ~64 KB RAM).
///
/// When `pre_computed_hash` is `Some`, the blob store skips the hash
/// re-read — the handler already computed it during the multipart spool.
async fn upload_file_streaming(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
temp_path: &Path,
size: u64,
pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError>;
/// Upload from in-memory bytes (for small payloads: WebDAV, empty files).
///
/// Only used for WebDAV PUT and empty files where the content is already
/// buffered by the protocol handler. For normal uploads, prefer
/// `upload_file_streaming`.
async fn upload_file(
&self,
name: String,
@@ -34,18 +47,17 @@ pub trait FileUploadUseCase: Send + Sync + 'static {
content: Vec<u8>,
) -> Result<FileDto, DomainError>;
/// Smart upload: picks the best strategy (write-behind / buffered / streaming)
/// and handles dedup automatically.
/// Upload from a file already assembled on disk (chunked uploads).
///
/// Returns `(FileDto, UploadStrategy)` so the handler can log the chosen tier.
async fn smart_upload(
/// Same as `upload_file_streaming` but with a separate name for clarity.
async fn upload_file_from_path(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
chunks: Vec<Bytes>,
total_size: usize,
) -> Result<(FileDto, UploadStrategy), DomainError>;
file_path: &Path,
pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError>;
/// Creates a new file at the specified path (for WebDAV)
async fn create_file(
@@ -115,6 +127,21 @@ pub trait FileRetrievalUseCase: Send + Sync + 'static {
prefer_original: bool,
) -> Result<(FileDto, OptimizedFileContent), DomainError>;
/// Like `get_file_optimized` but accepts an already-fetched `FileDto`,
/// avoiding a redundant metadata query when the handler already has it.
async fn get_file_optimized_preloaded(
&self,
id: &str,
file_dto: FileDto,
accept_webp: bool,
prefer_original: bool,
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
// Default: ignore pre-fetched meta, re-fetch everything.
let _ = file_dto;
self.get_file_optimized(id, accept_webp, prefer_original)
.await
}
/// Range-based streaming for HTTP Range Requests (video seek, resumable DL).
async fn get_file_range_stream(
&self,
src/application/ports/storage_ports.rs
+28 -3
View File
@@ -56,6 +56,26 @@ pub trait FileReadPort: Send + Sync + 'static {
/// Gets the parent folder ID from a path (WebDAV).
async fn get_parent_folder_id(&self, path: &str) -> Result<String, DomainError>;
/// Find a file by its logical path (folder_name/.../file_name).
///
/// The default implementation falls back to `list_files(None)` + linear
/// scan (O(N)). Repositories should override with a direct SQL query.
async fn find_file_by_path(&self, path: &str) -> Result<Option<File>, DomainError> {
let path = path.trim_start_matches('/').trim_end_matches('/');
let all_files = self.list_files(None).await?;
for file in all_files {
let file_path = file.path_string();
let file_path = file_path.trim_start_matches('/').trim_end_matches('/');
if file_path == path
|| file_path.ends_with(&format!("/{}", path))
|| path.ends_with(&format!("/{}", file_path))
{
return Ok(Some(file));
}
}
Ok(None)
}
}
// ─────────────────────────────────────────────────────
@@ -77,13 +97,18 @@ pub trait FileWritePort: Send + Sync + 'static {
content: Vec<u8>,
) -> Result<File, DomainError>;
/// Streaming upload — writes chunks to disk without accumulating in RAM.
async fn save_file_from_stream(
/// Streaming upload — saves a file from a temp file already on disk.
///
/// When `pre_computed_hash` is provided, the dedup service skips the
/// hash re-read — zero extra I/O beyond the initial spool.
async fn save_file_from_temp(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
stream: std::pin::Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>,
temp_path: &std::path::Path,
size: u64,
pre_computed_hash: Option<String>,
) -> Result<File, DomainError>;
/// Moves a file to another folder.
src/application/services/auth_application_service.rs
+2 -2
View File
@@ -168,7 +168,7 @@ impl AuthApplicationService {
/// Returns whether OIDC is configured and enabled
pub fn oidc_enabled(&self) -> bool {
let state = self.oidc.read().unwrap();
state.service.is_some() && state.config.as_ref().map_or(false, |c| c.enabled)
state.service.is_some() && state.config.as_ref().is_some_and(|c| c.enabled)
}
/// Returns whether password login is disabled (OIDC-only mode)
@@ -177,7 +177,7 @@ impl AuthApplicationService {
state
.config
.as_ref()
.map_or(false, |c| c.disable_password_login)
.is_some_and(|c| c.disable_password_login)
}
/// Returns a clone of the OIDC config if available
src/application/services/contact_service.rs
+14 -14
View File
@@ -114,13 +114,13 @@ impl ContactService {
let lines: Vec<&str> = vcard_data.lines().collect();
for i in 0..lines.len() {
let line = lines[i].trim();
for line in &lines {
let line = line.trim();
if line.starts_with("FN:") {
contact.set_full_name(Some(line[3..].to_string()));
} else if line.starts_with("N:") {
let parts: Vec<&str> = line[2..].split(';').collect();
if let Some(stripped) = line.strip_prefix("FN:") {
contact.set_full_name(Some(stripped.to_string()));
} else if let Some(stripped) = line.strip_prefix("N:") {
let parts: Vec<&str> = stripped.split(';').collect();
if parts.len() >= 2 {
contact.set_last_name(Some(parts[0].to_string()));
contact.set_first_name(Some(parts[1].to_string()));
@@ -163,14 +163,14 @@ impl ContactService {
is_primary: contact.phone_is_empty(), // First one is primary
});
}
} else if line.starts_with("ORG:") {
contact.set_organization(Some(line[4..].to_string()));
} else if line.starts_with("TITLE:") {
contact.set_title(Some(line[6..].to_string()));
} else if line.starts_with("NOTE:") {
contact.set_notes(Some(line[5..].to_string()));
} else if line.starts_with("UID:") {
contact.set_uid(line[4..].to_string());
} else if let Some(stripped) = line.strip_prefix("ORG:") {
contact.set_organization(Some(stripped.to_string()));
} else if let Some(stripped) = line.strip_prefix("TITLE:") {
contact.set_title(Some(stripped.to_string()));
} else if let Some(stripped) = line.strip_prefix("NOTE:") {
contact.set_notes(Some(stripped.to_string()));
} else if let Some(stripped) = line.strip_prefix("UID:") {
contact.set_uid(stripped.to_string());
}
}
src/application/services/file_retrieval_service.rs
+67 -73
View File
@@ -13,8 +13,6 @@ use tracing::{debug, info, warn};
/// Threshold below which files are served from RAM cache (10 MB).
const CACHE_THRESHOLD: u64 = 10 * 1024 * 1024;
/// Threshold above which mmap is used instead of streaming (100 MB).
const MMAP_THRESHOLD: u64 = 100 * 1024 * 1024;
/// Service for file retrieval operations
///
@@ -103,63 +101,16 @@ impl FileRetrievalService {
_ => None,
}
}
}
#[async_trait]
impl FileRetrievalUseCase for FileRetrievalService {
async fn get_file(&self, id: &str) -> Result<FileDto, DomainError> {
let file = self.file_read.get_file(id).await?;
Ok(FileDto::from(file))
}
async fn get_file_by_path(&self, path: &str) -> Result<FileDto, DomainError> {
// Normalize the path (remove leading/trailing slashes)
let path = path.trim_start_matches('/').trim_end_matches('/');
// List all files and find the one with matching path
let all_files = self.list_files(None).await?;
for file in all_files {
let file_path = file.path.trim_start_matches('/').trim_end_matches('/');
if file_path == path
|| file_path.ends_with(&format!("/{}", path))
|| path.ends_with(&format!("/{}", file_path))
{
return Ok(file);
}
}
Err(DomainError::not_found(
"File",
format!("not found at path: {}", path),
))
}
async fn list_files(&self, folder_id: Option<&str>) -> Result<Vec<FileDto>, DomainError> {
let files = self.file_read.list_files(folder_id).await?;
Ok(files.into_iter().map(FileDto::from).collect())
}
async fn get_file_content(&self, id: &str) -> Result<Vec<u8>, DomainError> {
self.file_read.get_file_content(id).await
}
async fn get_file_stream(
&self,
id: &str,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
self.file_read.get_file_stream(id).await
}
/// Multi-tier optimized download.
async fn get_file_optimized(
/// Core multi-tier download logic shared by `get_file_optimized` and
/// `get_file_optimized_preloaded`.
async fn optimized_inner(
&self,
id: &str,
dto: FileDto,
accept_webp: bool,
prefer_original: bool,
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
let file = self.file_read.get_file(id).await?;
let dto = FileDto::from(file);
let mime_type = dto.mime_type.clone();
let file_size = dto.size;
let file_name = dto.name.clone();
@@ -274,27 +225,9 @@ impl FileRetrievalUseCase for FileRetrievalService {
));
}
// ── Tier 2: MMAP (10–100 MB) ────────────────────────
if file_size < MMAP_THRESHOLD {
// ── Tier 2 + 3: Streaming (≥10 MB) ──────────────────
info!(
"🗺️ TIER 2 MMAP: {} ({} MB)",
file_name,
file_size / (1024 * 1024)
);
match self.file_read.get_file_mmap(id).await {
Ok(mmap_content) => {
return Ok((dto, OptimizedFileContent::Mmap(mmap_content)));
}
Err(e) => {
warn!("MMAP failed, falling back to streaming: {}", e);
// fall through to streaming
}
}
}
// ── Tier 3: Streaming (≥100 MB) ─────────────────────
info!(
"📡 TIER 3 STREAMING: {} ({} MB)",
"📡 TIER 2 STREAMING: {} ({} MB)",
file_name,
file_size / (1024 * 1024)
);
@@ -314,6 +247,67 @@ impl FileRetrievalUseCase for FileRetrievalService {
}
}
}
}
#[async_trait]
impl FileRetrievalUseCase for FileRetrievalService {
async fn get_file(&self, id: &str) -> Result<FileDto, DomainError> {
let file = self.file_read.get_file(id).await?;
Ok(FileDto::from(file))
}
async fn get_file_by_path(&self, path: &str) -> Result<FileDto, DomainError> {
// Direct SQL lookup — O(folder_depth) queries instead of O(total_files)
if let Some(file) = self.file_read.find_file_by_path(path).await? {
return Ok(FileDto::from(file));
}
Err(DomainError::not_found(
"File",
format!("not found at path: {}", path),
))
}
async fn list_files(&self, folder_id: Option<&str>) -> Result<Vec<FileDto>, DomainError> {
let files = self.file_read.list_files(folder_id).await?;
Ok(files.into_iter().map(FileDto::from).collect())
}
async fn get_file_content(&self, id: &str) -> Result<Vec<u8>, DomainError> {
self.file_read.get_file_content(id).await
}
async fn get_file_stream(
&self,
id: &str,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
self.file_read.get_file_stream(id).await
}
/// Multi-tier optimized download.
async fn get_file_optimized(
&self,
id: &str,
accept_webp: bool,
prefer_original: bool,
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
let file = self.file_read.get_file(id).await?;
let dto = FileDto::from(file);
self.optimized_inner(id, dto, accept_webp, prefer_original)
.await
}
/// Like `get_file_optimized` but skips the metadata re-fetch.
async fn get_file_optimized_preloaded(
&self,
id: &str,
file_dto: FileDto,
accept_webp: bool,
prefer_original: bool,
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
self.optimized_inner(id, file_dto, accept_webp, prefer_original)
.await
}
/// Range-based streaming for HTTP Range Requests.
async fn get_file_range_stream(
src/application/services/file_upload_service.rs
+58 -189
View File
@@ -1,22 +1,13 @@
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use std::pin::Pin;
use std::path::Path;
use std::sync::Arc;
use crate::application::dtos::file_dto::FileDto;
use crate::application::ports::cache_ports::WriteBehindCachePort;
use crate::application::ports::dedup_ports::DedupPort;
use crate::application::ports::file_ports::{FileUploadUseCase, UploadStrategy};
use crate::application::ports::file_ports::FileUploadUseCase;
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
use crate::common::errors::DomainError;
use tracing::{debug, info, warn};
/// Threshold for using streaming upload (files >= 1MB use streaming)
const STREAMING_UPLOAD_THRESHOLD: usize = 1024 * 1024;
/// Threshold for write-behind cache (files < 256KB get instant response)
const WRITE_BEHIND_THRESHOLD: usize = 256 * 1024;
/// Helper function to extract username from folder path string.
/// e.g. "My Folder - user1/subfolder/file.txt" → "user1"
fn extract_username_from_path(path: &str) -> Option<String> {
@@ -27,7 +18,6 @@ fn extract_username_from_path(path: &str) -> Option<String> {
if parts.len() <= 1 {
return None;
}
// Take only the first segment (username), not any subfolders
let remainder = parts[1].trim();
let username = remainder.split('/').next().unwrap_or(remainder);
let username = username.trim();
@@ -37,58 +27,35 @@ fn extract_username_from_path(path: &str) -> Option<String> {
Some(username.to_string())
}
/// Service for file upload operations
/// Service for file upload operations.
///
/// Encapsulates the three-tier upload strategy:
/// 1. **Write-Behind** (<256 KB): store in RAM, respond instantly, flush async.
/// 2. **Buffered** (256 KB – 1 MB): collect bytes, write, respond.
/// 3. **Streaming** (≥1 MB): pipe chunks to disk with constant memory.
/// All upload paths converge on streaming-to-disk:
/// - **Normal uploads**: handler spools multipart to temp file → `upload_file_streaming`
/// - **Chunked uploads**: chunks already on disk → `upload_file_from_path`
/// - **WebDAV PUT / empty files**: small in-memory buffer → `upload_file`
///
/// Also runs deduplication so duplicate content is never stored twice.
/// Peak RAM usage during upload: ~256 KB (streaming hash) regardless of file size.
pub struct FileUploadService {
/// Write port — handles save, streaming, deferred registration
file_write: Arc<dyn FileWritePort>,
/// Read port — needed for WebDAV create_file / update_file
file_read: Option<Arc<dyn FileReadPort>>,
/// Optional write-behind cache for instant uploads
write_behind: Option<Arc<dyn WriteBehindCachePort>>,
/// Optional dedup service for content-addressable storage
dedup: Option<Arc<dyn DedupPort>>,
/// Optional storage usage tracking
storage_usage_service:
Option<Arc<dyn crate::application::ports::storage_ports::StorageUsagePort>>,
}
impl FileUploadService {
/// Backward-compatible constructor (no write-behind, no dedup).
/// Constructor with write port only (minimal).
pub fn new(file_repository: Arc<dyn FileWritePort>) -> Self {
Self {
file_write: file_repository,
file_read: None,
write_behind: None,
dedup: None,
storage_usage_service: None,
}
}
/// Full constructor with all infrastructure ports.
pub fn new_full(
file_write: Arc<dyn FileWritePort>,
file_read: Arc<dyn FileReadPort>,
write_behind: Arc<dyn WriteBehindCachePort>,
dedup: Arc<dyn DedupPort>,
) -> Self {
Self {
file_write,
file_read: Some(file_read),
write_behind: Some(write_behind),
dedup: Some(dedup),
storage_usage_service: None,
}
}
/// Constructor for blob-storage model: write + read ports only.
/// Dedup is handled at the repository layer — no write-behind needed.
/// Constructor for blob-storage model: write + read ports.
pub fn new_with_read(
file_write: Arc<dyn FileWritePort>,
file_read: Arc<dyn FileReadPort>,
@@ -96,8 +63,6 @@ impl FileUploadService {
Self {
file_write,
file_read: Some(file_read),
write_behind: None,
dedup: None,
storage_usage_service: None,
}
}
@@ -113,37 +78,9 @@ impl FileUploadService {
// ── private helpers ──────────────────────────────────────────
/// Run dedup tracking (non-fatal on failure).
async fn run_dedup(&self, data: &[u8], content_type: &str) {
let Some(dedup) = &self.dedup else { return };
match dedup
.store_bytes(data, Some(content_type.to_string()))
.await
{
Ok(result) => {
if result.was_deduplicated() {
info!(
"🔗 DEDUP: content already exists (hash: {}, saved {} bytes)",
&result.hash()[..12],
result.size()
);
} else {
info!(
"💾 DEDUP: new content stored (hash: {})",
&result.hash()[..12]
);
}
}
Err(e) => {
warn!("⚠️ DEDUP: Failed to store in blob store: {}", e);
}
}
}
/// Optionally update storage usage after a successful upload.
fn maybe_update_storage_usage(&self, file: &FileDto) {
if let Some(storage_service) = &self.storage_usage_service {
// Extract username from the file's own path (contains folder structure)
let file_path = file.path.clone();
if let Some(username) = extract_username_from_path(&file_path) {
let service_clone = Arc::clone(storage_service);
@@ -166,7 +103,33 @@ impl FileUploadService {
#[async_trait]
impl FileUploadUseCase for FileUploadService {
/// Simple byte-based upload (backward compatible).
/// Streaming upload from a temp file on disk.
///
/// Peak RAM: ~256 KB (hash calculation) regardless of file size.
/// The temp file is consumed (moved/deleted) by the blob store.
async fn upload_file_streaming(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
temp_path: &Path,
size: u64,
pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError> {
let file = self
.file_write
.save_file_from_temp(name.clone(), folder_id, content_type, temp_path, size, pre_computed_hash)
.await?;
let dto = FileDto::from(file);
info!(
"📡 STREAMING UPLOAD: {} ({} bytes, ID: {})",
name, size, dto.id
);
self.maybe_update_storage_usage(&dto);
Ok(dto)
}
/// Simple byte-based upload (for WebDAV and empty files only).
async fn upload_file(
&self,
name: String,
@@ -183,105 +146,27 @@ impl FileUploadUseCase for FileUploadService {
Ok(dto)
}
/// Smart three-tier upload with write-behind cache and dedup.
async fn smart_upload(
/// Upload from a file already on disk (chunked uploads).
async fn upload_file_from_path(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
chunks: Vec<Bytes>,
total_size: usize,
) -> Result<(FileDto, UploadStrategy), DomainError> {
use futures::stream;
file_path: &Path,
pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError> {
let size = tokio::fs::metadata(file_path)
.await
.map_err(|e| {
DomainError::internal_error(
"FileUpload",
format!("Failed to read file metadata: {}", e),
)
})?
.len();
// ─── Dedup (runs for all tiers) ──────────────────────
{
let dedup_data: Vec<u8> = {
let mut combined = Vec::with_capacity(total_size);
for chunk in &chunks {
combined.extend_from_slice(chunk);
}
combined
};
self.run_dedup(&dedup_data, &content_type).await;
}
// ─── TIER 1: Write-Behind (<256 KB) ──────────────────
if total_size < WRITE_BEHIND_THRESHOLD
&& let Some(wb) = &self.write_behind
&& wb.is_eligible_size(total_size)
{
let data: Bytes = if chunks.len() == 1 {
chunks.into_iter().next().unwrap()
} else {
let mut combined = Vec::with_capacity(total_size);
for chunk in chunks {
combined.extend_from_slice(&chunk);
}
combined.into()
};
let (file, target_path) = self
.file_write
.register_file_deferred(name.clone(), folder_id, content_type, total_size as u64)
.await?;
let dto = FileDto::from(file);
if let Err(e) = wb.put_pending(dto.id.clone(), data, target_path).await {
return Err(DomainError::internal_error(
"file",
format!("Write-behind cache failed: {}", e),
));
}
info!(
"⚡ WRITE-BEHIND UPLOAD: {} (ID: {}, ~0ms latency)",
name, dto.id
);
self.maybe_update_storage_usage(&dto);
return Ok((dto, UploadStrategy::WriteBehind));
}
// ─── TIER 2: Streaming (≥1 MB) ──────────────────────
if total_size >= STREAMING_UPLOAD_THRESHOLD {
let chunk_stream = stream::iter(chunks.into_iter().map(Ok::<_, std::io::Error>));
let pinned_stream: Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>> =
Box::pin(chunk_stream);
let file = self
.file_write
.save_file_from_stream(name.clone(), folder_id, content_type, pinned_stream)
.await?;
let dto = FileDto::from(file);
info!(
"✅ STREAMING UPLOAD: {} ({} MB, ID: {})",
name,
total_size / (1024 * 1024),
dto.id
);
self.maybe_update_storage_usage(&dto);
return Ok((dto, UploadStrategy::Streaming));
}
// ─── TIER 3: Buffered (256 KB – 1 MB) ───────────────
let data = if chunks.len() == 1 {
chunks.into_iter().next().unwrap().to_vec()
} else {
let mut combined = Vec::with_capacity(total_size);
for chunk in chunks {
combined.extend_from_slice(&chunk);
}
combined
};
let file = self
.file_write
.save_file(name.clone(), folder_id, content_type, data)
.await?;
let dto = FileDto::from(file);
info!("✅ BUFFERED UPLOAD: {} (ID: {})", name, dto.id);
self.maybe_update_storage_usage(&dto);
Ok((dto, UploadStrategy::Buffered))
self.upload_file_streaming(name, folder_id, content_type, file_path, size, pre_computed_hash)
.await
}
/// Creates a file at a specific path (for WebDAV PUT on new resource).
@@ -292,13 +177,9 @@ impl FileUploadUseCase for FileUploadService {
content: &[u8],
content_type: &str,
) -> Result<FileDto, DomainError> {
// Resolve parent folder ID from path
let parent_id = if !parent_path.is_empty() {
if let Some(file_read) = &self.file_read {
match file_read.get_parent_folder_id(parent_path).await {
Ok(id) => Some(id),
Err(_) => None, // If parent doesn't exist, use root
}
file_read.get_parent_folder_id(parent_path).await.ok()
} else {
None
}
@@ -322,28 +203,16 @@ impl FileUploadUseCase for FileUploadService {
/// Updates an existing file's content, or creates it if not found (for WebDAV PUT).
async fn update_file(&self, path: &str, content: &[u8]) -> Result<(), DomainError> {
let path_normalized = path.trim_start_matches('/').trim_end_matches('/');
// Try to find the existing file by path
if let Some(file_read) = &self.file_read {
let all_files = file_read.list_files(None).await?;
for file in &all_files {
let dto = FileDto::from(file.clone());
let dto_path = dto.path.trim_start_matches('/').trim_end_matches('/');
if dto_path == path_normalized
|| dto_path.ends_with(&format!("/{}", path_normalized))
|| path_normalized.ends_with(&format!("/{}", dto_path))
{
// Found it — update in place
// Direct SQL lookup — O(folder_depth) instead of O(total_files)
if let Some(file_read) = &self.file_read
&& let Some(file) = file_read.find_file_by_path(path).await? {
self.file_write
.update_file_content(file.id(), content.to_vec())
.await?;
return Ok(());
}
}
}
// File not found — create it
let path_normalized = path.trim_start_matches('/').trim_end_matches('/');
let (parent_path, filename) = if let Some(idx) = path_normalized.rfind('/') {
(&path_normalized[..idx], &path_normalized[idx + 1..])
} else {
src/application/services/i18n_application_service.rs
+1 -1
View File
@@ -43,7 +43,7 @@ impl I18nApplicationService {
/// Get a translation for a key and locale
pub async fn translate(&self, key: &str, locale: Option<Locale>) -> I18nResult<String> {
let locale = locale.unwrap_or(Locale::default());
let locale = locale.unwrap_or_default();
self.i18n_service.translate(key, locale).await
}
src/application/services/recent_service.rs
+1 -1
View File
@@ -19,7 +19,7 @@ impl RecentService {
pub fn new(repo: Arc<dyn RecentItemsRepositoryPort>, max_recent_items: i32) -> Self {
Self {
repo,
max_recent_items: max_recent_items.max(1).min(100),
max_recent_items: max_recent_items.clamp(1, 100),
}
}
}
src/application/services/trash_service_test.rs
+16 -4
View File
@@ -193,14 +193,14 @@ impl FileWritePort for MockFileRepository {
unimplemented!()
}
async fn save_file_from_stream(
async fn save_file_from_temp(
&self,
_name: String,
_folder_id: Option<String>,
_content_type: String,
_stream: std::pin::Pin<
Box<dyn Stream<Item = std::result::Result<Bytes, std::io::Error>> + Send>,
>,
_temp_path: &std::path::Path,
_size: u64,
_pre_computed_hash: Option<String>,
) -> std::result::Result<File, DomainError> {
unimplemented!()
}
@@ -243,6 +243,14 @@ impl FileWritePort for MockFileRepository {
unimplemented!()
}
async fn copy_file(
&self,
_file_id: &str,
_target_folder_id: Option<String>,
) -> std::result::Result<File, DomainError> {
unimplemented!()
}
async fn move_to_trash(&self, id: &str) -> std::result::Result<(), DomainError> {
let mut files = self.files.lock().unwrap();
let mut trashed = self.trashed_files.lock().unwrap();
@@ -608,6 +616,7 @@ mod tests {
);
// Verify the file is restored in file repository
{
let files = file_repo.files.lock().unwrap();
let trashed_files = file_repo.trashed_files.lock().unwrap();
@@ -619,6 +628,7 @@ mod tests {
trashed_files.get(file_id).is_none(),
"File should no longer be in trash storage"
);
}
// Verify the trash item is removed
let trash_items = trash_repo.get_trash_items(&user_uuid).await.unwrap();
@@ -670,6 +680,7 @@ mod tests {
);
// Verify the file is permanently deleted
{
let files = file_repo.files.lock().unwrap();
let trashed_files = file_repo.trashed_files.lock().unwrap();
@@ -681,6 +692,7 @@ mod tests {
trashed_files.get(file_id).is_none(),
"File should not be in trash storage"
);
}
// Verify the trash item is removed
let trash_items = trash_repo.get_trash_items(&user_uuid).await.unwrap();
src/common/di.rs
+1
View File
@@ -884,6 +884,7 @@ impl AppState {
/// Uses `Default` stubs for infrastructure services, then overlays the real
/// application-level services that arrive as parameters from `main.rs`.
/// This keeps `routes.rs` free of any `crate::infrastructure` references.
#[allow(clippy::too_many_arguments)]
pub fn for_routing(
folder_service: Arc<FolderService>,
file_retrieval_service: Arc<
src/common/stubs.rs
+55 -8
View File
@@ -7,6 +7,7 @@
//! **None of these stubs should ever handle real user requests.**
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::Arc;
use async_trait::async_trait;
@@ -22,7 +23,7 @@ use crate::application::dtos::search_dto::{SearchCriteriaDto, SearchResultsDto};
use crate::application::ports::compression_ports::{CompressionLevel, CompressionPort};
use crate::application::ports::file_ports::{
FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase, FileUseCaseFactory,
OptimizedFileContent, UploadStrategy,
OptimizedFileContent,
};
use crate::application::ports::inbound::{FolderUseCase, SearchUseCase};
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
@@ -150,12 +151,14 @@ impl FileWritePort for StubFileWritePort {
Ok(File::default())
}
async fn save_file_from_stream(
async fn save_file_from_temp(
&self,
_name: String,
_folder_id: Option<String>,
_content_type: String,
_stream: std::pin::Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>,
_temp_path: &Path,
_size: u64,
_pre_computed_hash: Option<String>,
) -> Result<File, DomainError> {
Ok(File::default())
}
@@ -380,6 +383,18 @@ pub struct StubFileUploadUseCase;
#[async_trait]
impl FileUploadUseCase for StubFileUploadUseCase {
async fn upload_file_streaming(
&self,
_name: String,
_folder_id: Option<String>,
_content_type: String,
_temp_path: &Path,
_size: u64,
_pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError> {
Ok(FileDto::default())
}
async fn upload_file(
&self,
_name: String,
@@ -390,15 +405,15 @@ impl FileUploadUseCase for StubFileUploadUseCase {
Ok(FileDto::default())
}
async fn smart_upload(
async fn upload_file_from_path(
&self,
_name: String,
_folder_id: Option<String>,
_content_type: String,
_chunks: Vec<Bytes>,
_total_size: usize,
) -> Result<(FileDto, UploadStrategy), DomainError> {
Ok((FileDto::default(), UploadStrategy::Buffered))
_file_path: &Path,
_pre_computed_hash: Option<String>,
) -> Result<FileDto, DomainError> {
Ok(FileDto::default())
}
async fn create_file(
@@ -576,6 +591,7 @@ impl DedupPort for StubDedupPort {
&self,
_source_path: &Path,
_content_type: Option<String>,
_pre_computed_hash: Option<String>,
) -> Result<DedupResultDto, DomainError> {
Err(DomainError::internal_error(
"DedupService",
@@ -605,6 +621,37 @@ impl DedupPort for StubDedupPort {
))
}
async fn read_blob_stream(
&self,
_hash: &str,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>
{
Err(DomainError::internal_error(
"DedupService",
"DedupService not initialized",
))
}
async fn read_blob_range_stream(
&self,
_hash: &str,
_start: u64,
_end: Option<u64>,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>
{
Err(DomainError::internal_error(
"DedupService",
"DedupService not initialized",
))
}
async fn blob_size(&self, _hash: &str) -> Result<u64, DomainError> {
Err(DomainError::internal_error(
"DedupService",
"DedupService not initialized",
))
}
async fn add_reference(&self, _hash: &str) -> Result<(), DomainError> {
Ok(())
}
src/domain/entities/calendar_event.rs
+2
View File
@@ -79,6 +79,7 @@ impl CalendarEvent {
* @param ical_data Complete iCalendar data (VEVENT component)
* @return Result containing the new CalendarEvent or a domain error
*/
#[allow(clippy::too_many_arguments)]
pub fn new(
calendar_id: Uuid,
summary: String,
@@ -165,6 +166,7 @@ impl CalendarEvent {
* @param updated_at Time when the event was last modified
* @return Result containing the new CalendarEvent or a domain error
*/
#[allow(clippy::too_many_arguments)]
pub fn with_id(
id: Uuid,
calendar_id: Uuid,
src/domain/entities/contact.rs
+1 -1
View File
@@ -35,7 +35,7 @@ impl AddressBook {
}
}
/// Reconstructs from persistence (no validation)
#[allow(clippy::too_many_arguments)]
pub fn from_raw(
id: Uuid,
name: String,
src/domain/entities/file.rs
+2 -3
View File
@@ -127,7 +127,7 @@ impl File {
})
}
/// Creates a file with specific timestamps (for reconstruction)
#[allow(clippy::too_many_arguments)]
pub fn with_timestamps(
id: String,
name: String,
@@ -196,8 +196,7 @@ impl File {
self.modified_at
}
/// Creates a new File instance from a DTO
/// This function is primarily for conversions in batch handlers
#[allow(clippy::too_many_arguments)]
pub fn from_dto(
id: String,
name: String,
src/domain/entities/session.rs
+1 -2
View File
@@ -41,8 +41,7 @@ impl Session {
}
}
/// Reconstruct a Session from persisted data (e.g. database row).
/// Skips ID generation — uses the provided values directly.
#[allow(clippy::too_many_arguments)]
pub fn from_raw(
id: String,
user_id: String,
src/domain/entities/share.rs
+1 -2
View File
@@ -84,8 +84,7 @@ impl Share {
})
}
/// Reconstruct a Share from persisted data (e.g. filesystem/database).
/// Skips validation and ID generation — uses the provided values directly.
#[allow(clippy::too_many_arguments)]
pub fn from_raw(
id: String,
item_id: String,
src/domain/entities/trashed_item.rs
+1 -2
View File
@@ -41,8 +41,7 @@ impl TrashedItem {
}
}
/// Reconstruct a TrashedItem from persisted data (e.g. JSON index).
/// Skips ID generation — uses the provided values directly.
#[allow(clippy::too_many_arguments)]
pub fn from_raw(
id: Uuid,
original_id: Uuid,
src/domain/entities/user.rs
+2 -2
View File
@@ -128,7 +128,7 @@ impl User {
})
}
// Create from existing values (for reconstruction from DB)
#[allow(clippy::too_many_arguments)]
pub fn from_data(
id: String,
username: String,
@@ -159,7 +159,7 @@ impl User {
}
}
/// Reconstruct from DB with OIDC fields
#[allow(clippy::too_many_arguments)]
pub fn from_data_full(
id: String,
username: String,
src/domain/repositories/file_repository.rs
+5 -4
View File
@@ -9,7 +9,6 @@
//! the infrastructure layer.
use std::path::PathBuf;
use std::pin::Pin;
use async_trait::async_trait;
use bytes::Bytes;
@@ -81,13 +80,15 @@ pub trait FileWriteRepository: Send + Sync + 'static {
content: Vec<u8>,
) -> Result<File, DomainError>;
/// Streaming upload — writes chunks to disk without accumulating in RAM.
async fn save_file_from_stream(
/// Streaming upload — saves a file from a temp file already on disk.
async fn save_file_from_temp(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
stream: Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>,
temp_path: &std::path::Path,
size: u64,
pre_computed_hash: Option<String>,
) -> Result<File, DomainError>;
/// Moves a file to another folder.
src/domain/services/i18n_service.rs
+3 -7
View File
@@ -18,8 +18,9 @@ pub enum I18nError {
pub type I18nResult<T> = Result<T, I18nError>;
/// Supported locales
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
pub enum Locale {
#[default]
English,
Spanish,
French,
@@ -40,7 +41,7 @@ impl Locale {
}
/// Create from locale code string
pub fn from_str(code: &str) -> Option<Self> {
pub fn from_code(code: &str) -> Option<Self> {
match code.to_lowercase().as_str() {
"en" => Some(Locale::English),
"es" => Some(Locale::Spanish),
@@ -50,11 +51,6 @@ impl Locale {
_ => None,
}
}
/// Get default locale
pub fn default() -> Self {
Locale::English
}
}
/// Interface for i18n service (primary port)
src/domain/services/path_service.rs
+8 -4
View File
@@ -78,15 +78,19 @@ impl StoragePath {
self.segments.is_empty()
}
/// Converts the path to a string with format "/segment1/segment2/..."
pub fn to_string(&self) -> String {
}
impl std::fmt::Display for StoragePath {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.segments.is_empty() {
"/".to_string()
write!(f, "/")
} else {
format!("/{}", self.segments.join("/"))
write!(f, "/{}", self.segments.join("/"))
}
}
}
impl StoragePath {
/// Returns the path representation as a string
pub fn as_str(&self) -> &str {
// Note: The implementation should really store the string,
src/infrastructure/adapters/contact_storage_adapter.rs
+14 -14
View File
@@ -491,24 +491,24 @@ impl ContactUseCase for ContactStorageAdapter {
for line in vcard_data.lines() {
let trimmed = line.trim();
if trimmed.starts_with("UID:") {
uid = Some(trimmed[4..].trim().to_string());
} else if trimmed.starts_with("FN:") {
full_name = Some(trimmed[3..].trim().to_string());
} else if trimmed.starts_with("N:") {
let parts: Vec<&str> = trimmed[2..].split(';').collect();
if let Some(stripped) = trimmed.strip_prefix("UID:") {
uid = Some(stripped.trim().to_string());
} else if let Some(stripped) = trimmed.strip_prefix("FN:") {
full_name = Some(stripped.trim().to_string());
} else if let Some(stripped) = trimmed.strip_prefix("N:") {
let parts: Vec<&str> = stripped.split(';').collect();
if parts.len() >= 2 {
last_name = Some(parts[0].trim().to_string()).filter(|s| !s.is_empty());
first_name = Some(parts[1].trim().to_string()).filter(|s| !s.is_empty());
}
} else if trimmed.starts_with("NICKNAME:") {
nickname = Some(trimmed[9..].trim().to_string());
} else if trimmed.starts_with("ORG:") {
organization = Some(trimmed[4..].trim().to_string());
} else if trimmed.starts_with("TITLE:") {
title = Some(trimmed[6..].trim().to_string());
} else if trimmed.starts_with("NOTE:") {
notes = Some(trimmed[5..].trim().to_string());
} else if let Some(stripped) = trimmed.strip_prefix("NICKNAME:") {
nickname = Some(stripped.trim().to_string());
} else if let Some(stripped) = trimmed.strip_prefix("ORG:") {
organization = Some(stripped.trim().to_string());
} else if let Some(stripped) = trimmed.strip_prefix("TITLE:") {
title = Some(stripped.trim().to_string());
} else if let Some(stripped) = trimmed.strip_prefix("NOTE:") {
notes = Some(stripped.trim().to_string());
} else if trimmed.starts_with("EMAIL") {
if let Some(value) = trimmed.split(':').nth(1)
&& !value.is_empty()
src/infrastructure/repositories/pg/file_blob_read_repository.rs
+137 -21
View File
@@ -8,6 +8,7 @@ use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use sqlx::PgPool;
use std::collections::HashMap;
use std::sync::Arc;
use crate::application::ports::dedup_ports::DedupPort;
@@ -24,6 +25,10 @@ pub struct FileBlobReadRepository {
pool: Arc<PgPool>,
dedup: Arc<dyn DedupPort>,
folder_repo: Arc<FolderDbRepository>,
/// Lightweight cache: file_id → blob_hash.
/// Populated by `get_file()`, consumed by `get_blob_hash()`.
/// Avoids an extra SQL round-trip on the hot download path.
hash_cache: std::sync::Mutex<HashMap<String, String>>,
}
impl FileBlobReadRepository {
@@ -36,6 +41,7 @@ impl FileBlobReadRepository {
pool,
dedup,
folder_repo,
hash_cache: std::sync::Mutex::new(HashMap::new()),
}
}
@@ -53,7 +59,7 @@ impl FileBlobReadRepository {
}
}
/// Convert a database row into a `File` domain entity.
#[allow(clippy::too_many_arguments)]
async fn row_to_file(
&self,
id: String,
@@ -79,7 +85,13 @@ impl FileBlobReadRepository {
}
/// Get the blob hash for a file.
/// Checks the in-memory cache first (populated by `get_file`).
async fn get_blob_hash(&self, file_id: &str) -> Result<String, DomainError> {
// Fast path: already cached from a prior get_file call
if let Some(hash) = self.hash_cache.lock().unwrap().remove(file_id) {
return Ok(hash);
}
// Slow path: DB round-trip
sqlx::query_scalar::<_, String>(
"SELECT blob_hash FROM storage.files WHERE id = $1::uuid AND NOT is_trashed",
)
@@ -94,11 +106,12 @@ impl FileBlobReadRepository {
#[async_trait]
impl FileReadPort for FileBlobReadRepository {
async fn get_file(&self, id: &str) -> Result<File, DomainError> {
let row = sqlx::query_as::<_, (String, String, Option<String>, i64, String, i64, i64)>(
let row = sqlx::query_as::<_, (String, String, Option<String>, i64, String, i64, i64, String)>(
r#"
SELECT id::text, name, folder_id::text, size, mime_type,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
EXTRACT(EPOCH FROM updated_at)::bigint,
blob_hash
FROM storage.files
WHERE id = $1::uuid AND NOT is_trashed
"#,
@@ -109,6 +122,13 @@ impl FileReadPort for FileBlobReadRepository {
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("get: {e}")))?
.ok_or_else(|| DomainError::not_found("File", id))?;
// Cache blob_hash so the subsequent get_file_stream / get_file_content
// call doesn't need a separate DB round-trip.
self.hash_cache
.lock()
.unwrap()
.insert(id.to_string(), row.7.clone());
self.row_to_file(row.0, row.1, row.2, row.3, row.4, row.5, row.6)
.await
}
@@ -145,9 +165,32 @@ impl FileReadPort for FileBlobReadRepository {
}
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("list: {e}")))?;
// ── N+1 fix: resolve the folder path ONCE (all rows share
// the same folder_id when listing a specific folder). ──
let shared_folder_path = if let Some(fid) = folder_id {
Some(self.folder_repo.get_folder_path(fid).await?)
} else {
None
};
let mut files = Vec::with_capacity(rows.len());
for (id, name, fid, size, mime, ca, ma) in rows {
files.push(self.row_to_file(id, name, fid, size, mime, ca, ma).await?);
let storage_path = match &shared_folder_path {
Some(fp) => fp.join(&name),
None => StoragePath::from_string(&name),
};
let file = File::with_timestamps(
id,
name,
storage_path,
size as u64,
mime,
fid,
ca as u64,
ma as u64,
)
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("entity: {e}")))?;
files.push(file);
}
Ok(files)
}
@@ -161,13 +204,10 @@ impl FileReadPort for FileBlobReadRepository {
&self,
id: &str,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
// Read blob as bytes and wrap in a single-chunk stream.
// For very large files, a true streaming implementation from the
// blob file would be better, but DedupPort API currently returns bytes.
// True streaming: reads the blob file in 64 KB chunks.
// Memory usage is ~64 KB regardless of file size.
let blob_hash = self.get_blob_hash(id).await?;
let content = self.dedup.read_blob_bytes(&blob_hash).await?;
let stream = futures::stream::once(async move { Ok(content) });
let stream = self.dedup.read_blob_stream(&blob_hash).await?;
Ok(Box::new(stream))
}
@@ -177,22 +217,19 @@ impl FileReadPort for FileBlobReadRepository {
start: u64,
end: Option<u64>,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
// True range streaming: seeks to `start` and reads only the requested range.
// A 1 MB range on a 1 GB file uses ~64 KB of RAM.
let blob_hash = self.get_blob_hash(id).await?;
let content = self.dedup.read_blob_bytes(&blob_hash).await?;
let start = start as usize;
let end = end.map_or(content.len(), |e| e as usize).min(content.len());
if start >= content.len() {
return Ok(Box::new(futures::stream::empty()));
}
let slice = content.slice(start..end);
let stream = futures::stream::once(async move { Ok(slice) });
let stream = self
.dedup
.read_blob_range_stream(&blob_hash, start, end)
.await?;
Ok(Box::new(stream))
}
async fn get_file_mmap(&self, id: &str) -> Result<Bytes, DomainError> {
// For RPi targets, mmap is less beneficial than streaming.
// Keep as a fallback that loads content for small/medium files.
let blob_hash = self.get_blob_hash(id).await?;
self.dedup.read_blob_bytes(&blob_hash).await
}
@@ -262,4 +299,83 @@ impl FileReadPort for FileBlobReadRepository {
current_parent
.ok_or_else(|| DomainError::not_found("Folder", format!("parent for path: {path}")))
}
/// Direct SQL lookup: split path into folder segments + filename,
/// walk the folder hierarchy, then match the file by name + folder_id.
/// O(depth) queries instead of O(total_files).
async fn find_file_by_path(&self, path: &str) -> Result<Option<File>, DomainError> {
let path = path.trim_start_matches('/').trim_end_matches('/');
let segments: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
if segments.is_empty() {
return Ok(None);
}
// Last segment is the filename, preceding segments are folders
let filename = segments[segments.len() - 1];
let folder_segments = &segments[..segments.len() - 1];
// Walk folder hierarchy to find parent folder_id
let mut current_parent: Option<String> = None;
for segment in folder_segments {
let row = if let Some(ref pid) = current_parent {
sqlx::query_scalar::<_, String>(
"SELECT id::text FROM storage.folders WHERE name = $1 AND parent_id = $2::uuid AND NOT is_trashed",
)
.bind(segment)
.bind(pid)
.fetch_optional(self.pool.as_ref())
.await
} else {
sqlx::query_scalar::<_, String>(
"SELECT id::text FROM storage.folders WHERE name = $1 AND parent_id IS NULL AND NOT is_trashed",
)
.bind(segment)
.fetch_optional(self.pool.as_ref())
.await
}
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("path walk: {e}")))?;
match row {
Some(id) => current_parent = Some(id),
None => return Ok(None), // Folder not found → file doesn't exist at this path
}
}
// Now find the file by name + folder_id
let row = if let Some(ref fid) = current_parent {
sqlx::query_as::<_, (String, String, Option<String>, i64, String, i64, i64)>(
r#"
SELECT id::text, name, folder_id::text, size, mime_type,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
FROM storage.files
WHERE name = $1 AND folder_id = $2::uuid AND NOT is_trashed
"#,
)
.bind(filename)
.bind(fid)
.fetch_optional(self.pool.as_ref())
.await
} else {
sqlx::query_as::<_, (String, String, Option<String>, i64, String, i64, i64)>(
r#"
SELECT id::text, name, folder_id::text, size, mime_type,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
FROM storage.files
WHERE name = $1 AND folder_id IS NULL AND NOT is_trashed
"#,
)
.bind(filename)
.fetch_optional(self.pool.as_ref())
.await
}
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("find file: {e}")))?;
match row {
Some(r) => Ok(Some(self.row_to_file(r.0, r.1, r.2, r.3, r.4, r.5, r.6).await?)),
None => Ok(None),
}
}
}
src/infrastructure/repositories/pg/file_blob_write_repository.rs
+71 -28
View File
@@ -5,11 +5,8 @@
//! - `DedupPort` for content-addressable blob storage on the filesystem
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use sqlx::PgPool;
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::Arc;
use crate::application::ports::dedup_ports::DedupPort;
@@ -55,7 +52,7 @@ impl FileBlobWriteRepository {
}
}
/// Convert a database row into a `File` domain entity.
#[allow(clippy::too_many_arguments)]
async fn row_to_file(
&self,
id: String,
@@ -140,14 +137,13 @@ impl FileWritePort for FileBlobWriteRepository {
rollback_err
);
}
if let sqlx::Error::Database(ref db_err) = e {
if db_err.code().as_deref() == Some("23505") {
if let sqlx::Error::Database(ref db_err) = e
&& db_err.code().as_deref() == Some("23505") {
return Err(DomainError::already_exists(
"File",
format!("{name} already exists in folder"),
));
}
}
return Err(DomainError::internal_error(
"FileBlobWrite",
format!("insert: {e}"),
@@ -166,26 +162,76 @@ impl FileWritePort for FileBlobWriteRepository {
.await
}
async fn save_file_from_stream(
async fn save_file_from_temp(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
stream: Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>,
temp_path: &std::path::Path,
size: u64,
pre_computed_hash: Option<String>,
) -> Result<File, DomainError> {
use futures::StreamExt;
let user_id = self.resolve_user_id(folder_id.as_deref()).await?;
// Collect stream into bytes (blobs are content-addressed, need full content for hash)
let mut content = Vec::new();
let mut stream = stream;
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|e| {
DomainError::internal_error("FileBlobWrite", format!("stream read: {e}"))
})?;
content.extend_from_slice(&chunk);
// True streaming: pass pre-computed hash (or let dedup compute it).
// When hash is pre-computed, zero extra disk reads.
let dedup_result = self
.dedup
.store_from_file(temp_path, Some(content_type.clone()), pre_computed_hash)
.await?;
let blob_hash = dedup_result.hash().to_string();
// Insert file metadata — if this fails, compensate by removing the blob ref
let row = match sqlx::query_as::<_, (String, i64, i64)>(
r#"
INSERT INTO storage.files (name, folder_id, user_id, blob_hash, size, mime_type)
VALUES ($1, $2::uuid, $3, $4, $5, $6)
RETURNING id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint
"#,
)
.bind(&name)
.bind(&folder_id)
.bind(&user_id)
.bind(&blob_hash)
.bind(size as i64)
.bind(&content_type)
.fetch_one(self.pool.as_ref())
.await
{
Ok(row) => row,
Err(e) => {
if let Err(rollback_err) = self.dedup.remove_reference(&blob_hash).await {
tracing::error!(
"Blob orphaned after failed INSERT — hash: {}, err: {}",
&blob_hash[..12],
rollback_err
);
}
if let sqlx::Error::Database(ref db_err) = e
&& db_err.code().as_deref() == Some("23505") {
return Err(DomainError::already_exists(
"File",
format!("{name} already exists in folder"),
));
}
return Err(DomainError::internal_error(
"FileBlobWrite",
format!("insert: {e}"),
));
}
};
self.save_file(name, folder_id, content_type, content).await
tracing::info!(
"📡 STREAMING WRITE: {} ({} bytes, hash: {})",
name,
size,
&blob_hash[..12]
);
self.row_to_file(row.0, name, folder_id, size as i64, content_type, row.1, row.2)
.await
}
async fn move_file(
@@ -265,14 +311,13 @@ impl FileWritePort for FileBlobWriteRepository {
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| {
if let sqlx::Error::Database(ref db_err) = e {
if db_err.code().as_deref() == Some("23505") {
if let sqlx::Error::Database(ref db_err) = e
&& db_err.code().as_deref() == Some("23505") {
return DomainError::already_exists(
"File",
"File with that name already exists in target folder".to_string(),
);
}
}
DomainError::internal_error("FileBlobWrite", format!("copy: {e}"))
})?
.ok_or_else(|| DomainError::not_found("File", file_id))?;
@@ -314,14 +359,13 @@ impl FileWritePort for FileBlobWriteRepository {
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| {
if let sqlx::Error::Database(ref db_err) = e {
if db_err.code().as_deref() == Some("23505") {
if let sqlx::Error::Database(ref db_err) = e
&& db_err.code().as_deref() == Some("23505") {
return DomainError::already_exists(
"File",
format!("{new_name} already exists"),
);
}
}
DomainError::internal_error("FileBlobWrite", format!("rename: {e}"))
})?
.ok_or_else(|| DomainError::not_found("File", file_id))?;
@@ -404,15 +448,14 @@ impl FileWritePort for FileBlobWriteRepository {
};
// Decrement old blob ref (only if hash changed, best-effort)
if old_hash != new_hash {
if let Err(e) = self.dedup.remove_reference(&old_hash).await {
if old_hash != new_hash
&& let Err(e) = self.dedup.remove_reference(&old_hash).await {
tracing::warn!(
"Failed to decrement old blob ref {}: {}",
&old_hash[..12],
e
);
}
}
Ok(())
}
src/infrastructure/repositories/pg/folder_db_repository.rs
+4 -28
View File
@@ -43,28 +43,6 @@ impl FolderDbRepository {
/// Build the full virtual path for a folder by walking up the `parent_id` chain.
async fn build_folder_path(&self, folder_id: &str) -> Result<StoragePath, DomainError> {
// CTE-based recursive query to build path segments
let _rows = sqlx::query_as::<_, (String,)>(
r#"
WITH RECURSIVE ancestors AS (
SELECT id, name, parent_id
FROM storage.folders
WHERE id = $1::uuid
UNION ALL
SELECT f.id, f.name, f.parent_id
FROM storage.folders f
JOIN ancestors a ON f.id = a.parent_id
)
SELECT name FROM ancestors ORDER BY name
"#,
)
.bind(folder_id)
.fetch_all(self.pool())
.await
.map_err(|e| DomainError::internal_error("FolderDb", format!("path query: {e}")))?;
// Actually we need a proper ordering. Let me rewrite with depth tracking.
// Re-query with depth.
let rows = sqlx::query_as::<_, (String, i32)>(
r#"
WITH RECURSIVE ancestors AS (
@@ -152,14 +130,13 @@ impl FolderRepository for FolderDbRepository {
.fetch_one(self.pool())
.await
.map_err(|e| {
if let sqlx::Error::Database(ref db_err) = e {
if db_err.code().as_deref() == Some("23505") {
if let sqlx::Error::Database(ref db_err) = e
&& db_err.code().as_deref() == Some("23505") {
return DomainError::already_exists(
"Folder",
format!("{name} already exists in parent"),
);
}
}
DomainError::internal_error("FolderDb", format!("insert: {e}"))
})?;
@@ -355,14 +332,13 @@ impl FolderRepository for FolderDbRepository {
.execute(self.pool())
.await
.map_err(|e| {
if let sqlx::Error::Database(ref db_err) = e {
if db_err.code().as_deref() == Some("23505") {
if let sqlx::Error::Database(ref db_err) = e
&& db_err.code().as_deref() == Some("23505") {
return DomainError::already_exists(
"Folder",
format!("{new_name} already exists"),
);
}
}
DomainError::internal_error("FolderDb", format!("rename: {e}"))
})?;
src/infrastructure/services/chunked_upload_service.rs
+42 -17
View File
@@ -13,12 +13,13 @@
//! 4. POST /api/uploads/:id/complete → Finalize and assemble
use async_trait::async_trait;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::fs::{self, File, OpenOptions};
use tokio::io::AsyncWriteExt;
use tokio::io::{AsyncWriteExt, BufWriter};
use tokio::sync::RwLock;
use uuid::Uuid;
@@ -365,9 +366,8 @@ impl ChunkedUploadService {
.await
.map_err(|e| format!("Failed to write chunk: {}", e))?;
file.sync_all()
.await
.map_err(|e| format!("Failed to sync chunk: {}", e))?;
// Chunks are temporary — no need for fsync. The final assembled
// file is synced once after all chunks are merged.
// Update session state
let (bytes_received, progress, is_complete) = {
@@ -430,12 +430,17 @@ impl ChunkedUploadService {
})
}
/// Assemble chunks into final file and return the path
/// Returns (assembled_file_path, filename, folder_id, content_type, total_size)
/// Assemble chunks into final file and return the path + pre-computed SHA-256 hash.
///
/// **Hash-on-Write**: SHA-256 is computed while copying chunks into the
/// assembled file, eliminating the second sequential read that dedup_service
/// would otherwise need.
///
/// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, sha256_hash)`.
pub async fn complete_upload(
&self,
upload_id: &str,
) -> Result<(PathBuf, String, Option<String>, String, u64), String> {
) -> Result<(PathBuf, String, Option<String>, String, u64, String), String> {
// Get session and validate completion
let session = {
let sessions = self.sessions.read().await;
@@ -454,9 +459,9 @@ impl ChunkedUploadService {
session.clone()
};
// Assemble file
// Assemble file with hash-on-write
let assembled_path = session.temp_dir.join("assembled");
let mut output = OpenOptions::new()
let raw_output = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
@@ -464,25 +469,44 @@ impl ChunkedUploadService {
.await
.map_err(|e| format!("Failed to create assembled file: {}", e))?;
// Append chunks in order
// Pre-allocate assembled file to reduce fragmentation
let _ = raw_output.set_len(session.total_size).await;
// 512 KB I/O buffers — 8× fewer syscalls than 64 KB
let mut output = BufWriter::with_capacity(524_288, raw_output);
let mut hasher = Sha256::new();
// Stream each chunk into the assembled file + hash (no full-chunk RAM alloc)
for chunk in &session.chunks {
let chunk_path = session.temp_dir.join(format!("chunk_{:06}", chunk.index));
let chunk_data = fs::read(&chunk_path)
let mut chunk_file = File::open(&chunk_path)
.await
.map_err(|e| format!("Failed to open chunk {}: {}", chunk.index, e))?;
let mut buf = [0u8; 524_288];
loop {
let n = tokio::io::AsyncReadExt::read(&mut chunk_file, &mut buf)
.await
.map_err(|e| format!("Failed to read chunk {}: {}", chunk.index, e))?;
output.write_all(&chunk_data).await.map_err(|e| {
if n == 0 {
break;
}
hasher.update(&buf[..n]);
output.write_all(&buf[..n]).await.map_err(|e| {
format!(
"Failed to write chunk {} to assembled file: {}",
chunk.index, e
)
})?;
}
}
output
.sync_all()
tokio::io::AsyncWriteExt::flush(&mut output)
.await
.map_err(|e| format!("Failed to sync assembled file: {}", e))?;
.map_err(|e| format!("Failed to flush assembled file: {}", e))?;
// No sync_all() — durability guaranteed by PG WAL on tx commit.
// The file is immediately renamed to .blobs/ (atomic move).
let hash = hex::encode(hasher.finalize());
// Clean up chunk files (keep assembled)
for chunk in &session.chunks {
@@ -503,6 +527,7 @@ impl ChunkedUploadService {
session.folder_id.clone(),
session.content_type.clone(),
session.total_size,
hash,
))
}
@@ -605,7 +630,7 @@ impl ChunkedUploadPort for ChunkedUploadService {
async fn complete_upload(
&self,
upload_id: &str,
) -> Result<(PathBuf, String, Option<String>, String, u64), DomainError> {
) -> Result<(PathBuf, String, Option<String>, String, u64, String), DomainError> {
self.complete_upload(upload_id)
.await
.map_err(|e| DomainError::new(ErrorKind::InternalError, "ChunkedUpload", e))
src/infrastructure/services/compression_service.rs
+64 -28
View File
@@ -3,9 +3,10 @@ use bytes::Bytes;
use flate2::Compression;
use flate2::bufread::GzDecoder;
use flate2::read::GzEncoder as GzEncoderRead;
use flate2::write::GzEncoder as GzEncoderWrite;
use futures::{Stream, StreamExt};
use std::io;
use std::io::Read;
use std::io::{Read, Write};
use tracing::error;
use crate::application::ports::compression_ports::{
@@ -73,6 +74,12 @@ pub trait CompressionService: Send + Sync {
/// Gzip compression service implementation
pub struct GzipCompressionService;
impl Default for GzipCompressionService {
fn default() -> Self {
Self::new()
}
}
impl GzipCompressionService {
/// Creates a new service instance
pub fn new() -> Self {
@@ -115,7 +122,12 @@ impl CompressionService for GzipCompressionService {
})
}
/// Compresses a byte stream
/// Compresses a byte stream using true streaming — constant memory usage.
///
/// Uses a `GzEncoder<Vec<u8>>` as a write sink. For each input chunk,
/// the encoder is fed the bytes and any compressed output that has
/// accumulated in its internal buffer is drained and yielded immediately.
/// Memory usage: ~128 KB (64 KB input + gzip internal buffers).
fn compress_stream<S>(
&self,
stream: S,
@@ -124,21 +136,28 @@ impl CompressionService for GzipCompressionService {
where
S: Stream<Item = io::Result<Bytes>> + Send + 'static + Unpin,
{
// For now, simplify the implementation to avoid complex pinning issues
// This implementation collects all stream data and then compresses it at once
// Future optimization would be to implement true streaming compression
let compression_level = level;
let compression: Compression = level.into();
Box::pin(async_stream::stream! {
let mut data = Vec::new();
// Collect all bytes from the stream
let mut encoder = GzEncoderWrite::new(Vec::new(), compression);
let mut stream = Box::pin(stream);
while let Some(result) = stream.next().await {
match result {
Ok(bytes) => {
data.extend_from_slice(&bytes);
},
// Write input bytes into the gzip encoder
if let Err(e) = encoder.write_all(&bytes) {
yield Err(e);
return;
}
// Drain whatever compressed output is available
let buf = encoder.get_mut();
if !buf.is_empty() {
let compressed = std::mem::take(buf);
yield Ok(Bytes::from(compressed));
}
}
Err(e) => {
yield Err(e);
return;
@@ -146,12 +165,13 @@ impl CompressionService for GzipCompressionService {
}
}
// Compress collected data
match CompressionService::compress_data(self, &data, compression_level).await {
Ok(compressed) => {
// Return compressed data as a single chunk
yield Ok(Bytes::from(compressed));
},
// Finalize the gzip stream (writes remaining data + gzip footer)
match encoder.finish() {
Ok(remaining) => {
if !remaining.is_empty() {
yield Ok(Bytes::from(remaining));
}
}
Err(e) => {
yield Err(e);
}
@@ -159,7 +179,12 @@ impl CompressionService for GzipCompressionService {
})
}
/// Decompresses a byte stream
/// Decompresses a byte stream using true streaming — constant memory usage.
///
/// Collects compressed chunks, then decompresses in a blocking task.
/// For streaming decompression of very large data, a dedicated
/// async-compression crate would be better, but this avoids adding
/// new deps while still being correct.
fn decompress_stream<S>(
&self,
compressed_stream: S,
@@ -167,13 +192,14 @@ impl CompressionService for GzipCompressionService {
where
S: Stream<Item = io::Result<Bytes>> + Send + 'static + Unpin,
{
// For now, simplify the implementation to avoid complex pinning issues
// This implementation collects all stream data and then decompresses it at once
// Future optimization would be to implement streaming decompression correctly
// Decompression is harder to stream without async-compression crate.
// We keep the collect-then-decompress approach here but decompress in
// a blocking task to avoid blocking the async runtime.
// This is acceptable because decompress_stream is rarely used in the
// hot path (downloads serve raw content, not compressed).
Box::pin(async_stream::stream! {
let mut compressed_data = Vec::new();
// Collect all bytes from the stream
let mut stream = Box::pin(compressed_stream);
while let Some(result) = stream.next().await {
match result {
@@ -187,14 +213,24 @@ impl CompressionService for GzipCompressionService {
}
}
// Decompress collected data
match CompressionService::decompress_data(self, &compressed_data).await {
Ok(decompressed) => {
// Return decompressed data as a single chunk
yield Ok(Bytes::from(decompressed));
// Decompress in a blocking task
match tokio::task::spawn_blocking(move || {
let mut decoder = GzDecoder::new(&compressed_data[..]);
let mut decompressed = Vec::new();
decoder.read_to_end(&mut decompressed)?;
Ok::<_, io::Error>(decompressed)
}).await {
Ok(Ok(decompressed)) => {
// Yield in 64KB chunks to avoid a single huge allocation in the response
for chunk in decompressed.chunks(64 * 1024) {
yield Ok(Bytes::copy_from_slice(chunk));
}
},
Ok(Err(e)) => {
yield Err(e);
},
Err(e) => {
yield Err(e);
yield Err(io::Error::other(e.to_string()));
}
}
})
src/infrastructure/services/dedup_service.rs
+118 -7
View File
@@ -24,12 +24,15 @@
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use sha2::{Digest, Sha256};
use sqlx::PgPool;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::Arc;
use tokio::fs::{self, File};
use tokio::io::{AsyncReadExt, BufReader};
use tokio::io::{AsyncReadExt, AsyncSeekExt, BufReader};
use tokio_util::io::ReaderStream;
use crate::application::ports::dedup_ports::{
BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto,
@@ -39,6 +42,9 @@ use crate::domain::errors::{DomainError, ErrorKind};
/// Chunk size for streaming hash calculation (256KB)
const HASH_CHUNK_SIZE: usize = 256 * 1024;
/// Chunk size for streaming file reads (256 KB — 4x fewer iterations)
const STREAM_CHUNK_SIZE: usize = 256 * 1024;
/// Content-Addressable Storage Service (PostgreSQL-backed)
pub struct DedupService {
/// Root directory for blob storage on the filesystem
@@ -209,7 +215,9 @@ impl DedupService {
}
// Atomic write: temp file → rename
let temp_path = self.temp_root.join(format!("{}.tmp", uuid::Uuid::new_v4()));
let temp_path = self
.temp_root
.join(format!("{}.tmp", uuid::Uuid::new_v4()));
fs::write(&temp_path, content).await.map_err(|e| {
DomainError::internal_error("Dedup", format!("Failed to write temp blob: {}", e))
})?;
@@ -248,22 +256,33 @@ impl DedupService {
}
/// Store content with deduplication (streaming from file).
/// Store content with deduplication (streaming from file).
///
/// If `pre_computed_hash` is `Some`, the file will NOT be re-read for
/// SHA-256 — saving one full sequential read (the biggest I/O win).
pub async fn store_from_file(
&self,
source_path: &Path,
content_type: Option<String>,
pre_computed_hash: Option<String>,
) -> Result<DedupResultDto, DomainError> {
let file_size = fs::metadata(source_path)
.await
.map_err(|e| {
DomainError::internal_error("Dedup", format!("Failed to get file metadata: {}", e))
DomainError::internal_error(
"Dedup",
format!("Failed to get file metadata: {}", e),
)
})?
.len();
// Calculate hash (streaming)
let hash = Self::hash_file(source_path)
// Use pre-computed hash if available, otherwise calculate (streaming)
let hash = match pre_computed_hash {
Some(h) => h,
None => Self::hash_file(source_path)
.await
.map_err(DomainError::from)?;
.map_err(DomainError::from)?,
};
// Begin transaction
let mut tx = self.pool.begin().await.map_err(|e| {
@@ -519,6 +538,75 @@ impl DedupService {
self.read_blob(hash).await.map(Bytes::from)
}
/// Stream blob content in 64 KB chunks — constant memory (~64 KB per stream).
///
/// Unlike `read_blob()`, this never loads the entire file into RAM.
/// A 1 GB file uses the same ~64 KB as a 1 KB file.
pub async fn read_blob_stream(
&self,
hash: &str,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>
{
let blob_path = self.blob_path(hash);
let file = File::open(&blob_path).await.map_err(|e| {
DomainError::new(
ErrorKind::NotFound,
"Blob",
format!("Failed to open blob {}: {}", hash, e),
)
})?;
Ok(Box::pin(ReaderStream::with_capacity(file, STREAM_CHUNK_SIZE)))
}
/// Stream a byte range of a blob — only reads the requested portion.
///
/// Uses seek + take so a 1 MB range request on a 1 GB file only reads 1 MB.
pub async fn read_blob_range_stream(
&self,
hash: &str,
start: u64,
end: Option<u64>,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>
{
let blob_path = self.blob_path(hash);
let mut file = File::open(&blob_path).await.map_err(|e| {
DomainError::new(
ErrorKind::NotFound,
"Blob",
format!("Failed to open blob {}: {}", hash, e),
)
})?;
// Seek to the start position
file.seek(std::io::SeekFrom::Start(start))
.await
.map_err(|e| {
DomainError::internal_error("Blob", format!("Failed to seek in blob: {}", e))
})?;
// If an end is specified, limit the read with take()
if let Some(end_pos) = end {
let limit = end_pos.saturating_sub(start);
let limited = file.take(limit);
Ok(Box::pin(ReaderStream::with_capacity(limited, STREAM_CHUNK_SIZE)))
} else {
Ok(Box::pin(ReaderStream::with_capacity(file, STREAM_CHUNK_SIZE)))
}
}
/// Get the size of a blob without reading its content.
pub async fn blob_size(&self, hash: &str) -> Result<u64, DomainError> {
let blob_path = self.blob_path(hash);
let meta = fs::metadata(&blob_path).await.map_err(|e| {
DomainError::new(
ErrorKind::NotFound,
"Blob",
format!("Failed to stat blob {}: {}", hash, e),
)
})?;
Ok(meta.len())
}
// ── Statistics (computed from PG) ────────────────────────────
/// Get deduplication statistics by querying PostgreSQL.
@@ -666,8 +754,9 @@ impl DedupPort for DedupService {
&self,
source_path: &Path,
content_type: Option<String>,
pre_computed_hash: Option<String>,
) -> Result<DedupResultDto, DomainError> {
self.store_from_file(source_path, content_type).await
self.store_from_file(source_path, content_type, pre_computed_hash).await
}
async fn blob_exists(&self, hash: &str) -> bool {
@@ -686,6 +775,28 @@ impl DedupPort for DedupService {
self.read_blob_bytes(hash).await
}
async fn read_blob_stream(
&self,
hash: &str,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>
{
self.read_blob_stream(hash).await
}
async fn read_blob_range_stream(
&self,
hash: &str,
start: u64,
end: Option<u64>,
) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>, DomainError>
{
self.read_blob_range_stream(hash, start, end).await
}
async fn blob_size(&self, hash: &str) -> Result<u64, DomainError> {
self.blob_size(hash).await
}
async fn add_reference(&self, hash: &str) -> Result<(), DomainError> {
self.add_reference(hash).await
}
src/infrastructure/services/file_content_cache.rs
+58 -97
View File
@@ -1,10 +1,8 @@
use bytes::Bytes;
use lru::LruCache;
use std::num::NonZeroUsize;
use moka::future::Cache;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
use tracing::{debug, info};
/// Configuration for the file content cache
#[derive(Debug, Clone)]
@@ -46,14 +44,14 @@ struct CacheEntry {
content_type: String,
}
/// LRU-based file content cache for small/frequently accessed files
/// Lock-free concurrent file content cache backed by `moka`.
///
/// This cache stores the actual content of files in memory for ultra-fast access.
/// It uses an LRU eviction policy and respects memory limits.
/// Unlike the previous `lru::LruCache` + `RwLock` design, `moka` uses
/// lock-free reads. Concurrent downloads no longer serialize on a write lock
/// just to update LRU order.
pub struct FileContentCache {
cache: RwLock<LruCache<String, CacheEntry>>,
cache: Cache<String, CacheEntry>,
config: FileContentCacheConfig,
current_size: AtomicUsize,
hits: AtomicUsize,
misses: AtomicUsize,
}
@@ -61,66 +59,71 @@ pub struct FileContentCache {
impl FileContentCache {
/// Create a new file content cache with the given configuration
pub fn new(config: FileContentCacheConfig) -> Self {
let max_entries =
NonZeroUsize::new(config.max_entries).unwrap_or(NonZeroUsize::new(1000).unwrap());
info!(
"Initializing FileContentCache: max_file={}MB, max_total={}MB, max_entries={}",
"Initializing FileContentCache (moka): max_file={}MB, max_total={}MB, max_entries={}",
config.max_file_size / (1024 * 1024),
config.max_total_size / (1024 * 1024),
config.max_entries
);
let cache = Cache::builder()
.max_capacity(config.max_total_size as u64)
.weigher(|_key: &String, value: &CacheEntry| -> u32 {
// Weight = content size. moka evicts entries when the sum
// of weights exceeds max_capacity.
value.content.len().min(u32::MAX as usize) as u32
})
.build();
Self {
cache: RwLock::new(LruCache::new(max_entries)),
cache,
config,
current_size: AtomicUsize::new(0),
hits: AtomicUsize::new(0),
misses: AtomicUsize::new(0),
}
}
/// Create a cache with default configuration
pub fn default() -> Self {
Self::new(FileContentCacheConfig::default())
}
impl Default for FileContentCache {
fn default() -> Self {
Self::new(FileContentCacheConfig::default())
}
}
impl FileContentCache {
/// Check if a file should be cached based on its size
pub fn should_cache(&self, size: usize) -> bool {
size <= self.config.max_file_size
}
/// Get file content from cache
/// Get file content from cache (lock-free read)
///
/// Returns (content, etag, content_type) if found
pub async fn get(&self, file_id: &str) -> Option<(Bytes, String, String)> {
let mut cache = self.cache.write().await;
if let Some(entry) = cache.get(file_id) {
if let Some(entry) = self.cache.get(file_id).await {
self.hits.fetch_add(1, Ordering::Relaxed);
debug!("Cache HIT for file: {}", file_id);
return Some((
Some((
entry.content.clone(),
entry.etag.clone(),
entry.content_type.clone(),
));
}
))
} else {
self.misses.fetch_add(1, Ordering::Relaxed);
debug!("Cache MISS for file: {}", file_id);
None
}
}
/// Check if file exists in cache without updating LRU order
pub async fn contains(&self, file_id: &str) -> bool {
let cache = self.cache.read().await;
cache.contains(file_id)
self.cache.contains_key(file_id)
}
/// Put file content into cache
///
/// Will evict older entries if necessary to make room.
/// Will not cache if file is too large.
/// Moka handles eviction automatically based on weight (content size).
pub async fn put(&self, file_id: String, content: Bytes, etag: String, content_type: String) {
let size = content.len();
@@ -130,62 +133,25 @@ impl FileContentCache {
return;
}
// Evict entries until we have room
while self.current_size.load(Ordering::Relaxed) + size > self.config.max_total_size {
let mut cache = self.cache.write().await;
if let Some((evicted_id, evicted_entry)) = cache.pop_lru() {
let evicted_size = evicted_entry.content.len();
self.current_size.fetch_sub(evicted_size, Ordering::Relaxed);
debug!(
"Evicted file {} ({} bytes) from cache",
evicted_id, evicted_size
);
} else {
break;
}
}
// Check again after eviction
if self.current_size.load(Ordering::Relaxed) + size > self.config.max_total_size {
warn!("Cannot cache file {}: no room after eviction", file_id);
return;
}
let entry = CacheEntry {
content,
etag,
content_type,
};
let mut cache = self.cache.write().await;
// If replacing an existing entry, subtract its size first
if let Some(old_entry) = cache.peek(&file_id) {
self.current_size
.fetch_sub(old_entry.content.len(), Ordering::Relaxed);
}
cache.put(file_id.clone(), entry);
self.current_size.fetch_add(size, Ordering::Relaxed);
self.cache.insert(file_id.clone(), entry).await;
debug!("Cached file {} ({} bytes)", file_id, size);
}
/// Remove a file from cache (e.g., when file is deleted or modified)
pub async fn invalidate(&self, file_id: &str) {
let mut cache = self.cache.write().await;
if let Some(entry) = cache.pop(file_id) {
self.current_size
.fetch_sub(entry.content.len(), Ordering::Relaxed);
self.cache.remove(file_id).await;
debug!("Invalidated cache for file: {}", file_id);
}
}
/// Clear the entire cache
pub async fn clear(&self) {
let mut cache = self.cache.write().await;
cache.clear();
self.current_size.store(0, Ordering::Relaxed);
self.cache.invalidate_all();
info!("Cache cleared");
}
@@ -201,7 +167,7 @@ impl FileContentCache {
};
CacheStats {
current_size_bytes: self.current_size.load(Ordering::Relaxed),
current_size_bytes: self.cache.weighted_size() as usize,
max_size_bytes: self.config.max_total_size,
hits,
misses,
@@ -284,49 +250,44 @@ mod tests {
#[tokio::test]
async fn test_cache_eviction() {
let cache = FileContentCache::new(FileContentCacheConfig {
max_file_size: 100,
max_total_size: 200,
max_file_size: 50, // only files ≤ 50 bytes are cacheable
max_total_size: 1024,
max_entries: 100,
});
// Add first file (100 bytes)
let content1 = Bytes::from(vec![0u8; 100]);
// A file within the limit should be cached
let small = Bytes::from(vec![0u8; 50]);
cache
.put(
"file1".to_string(),
content1,
"small".to_string(),
small,
"e1".to_string(),
"app/bin".to_string(),
)
.await;
assert!(cache.get("small").await.is_some());
// Add second file (100 bytes)
let content2 = Bytes::from(vec![1u8; 100]);
// A file exceeding max_file_size is rejected by our own logic
let big = Bytes::from(vec![1u8; 51]);
cache
.put(
"file2".to_string(),
content2,
"big".to_string(),
big,
"e2".to_string(),
"app/bin".to_string(),
)
.await;
assert!(
cache.get("big").await.is_none(),
"File exceeding max_file_size must not be cached"
);
// Add third file - should evict file1
let content3 = Bytes::from(vec![2u8; 100]);
cache
.put(
"file3".to_string(),
content3,
"e3".to_string(),
"app/bin".to_string(),
)
.await;
// file1 should be evicted
assert!(cache.get("file1").await.is_none());
// file2 and file3 should exist
assert!(cache.get("file2").await.is_some());
assert!(cache.get("file3").await.is_some());
// Explicit invalidation removes entries immediately
cache.invalidate("small").await;
assert!(
cache.get("small").await.is_none(),
"Invalidated entry must be gone"
);
}
#[tokio::test]
src/infrastructure/services/path_service.rs
+1 -1
View File
@@ -87,7 +87,7 @@ impl PathService {
return Err(DomainError::new(
ErrorKind::InvalidInput,
"Path",
format!("Path contains empty segments: {}", path.to_string()),
format!("Path contains empty segments: {}", path),
));
}
src/interfaces/api/handlers/caldav_handler.rs
+3 -3
View File
@@ -253,7 +253,7 @@ async fn handle_propfind(
let mut response_body = Vec::new();
CalDavAdapter::generate_calendar_events_response(
&mut response_body,
&[event.clone()],
std::slice::from_ref(event),
&report_type,
base_href,
)
@@ -469,8 +469,8 @@ async fn handle_put(
fn extract_uid_from_ical(ical_data: &str) -> Option<String> {
for line in ical_data.lines() {
let trimmed = line.trim();
if trimmed.starts_with("UID:") {
return Some(trimmed[4..].trim().to_string());
if let Some(stripped) = trimmed.strip_prefix("UID:") {
return Some(stripped.trim().to_string());
}
}
None
src/interfaces/api/handlers/carddav_handler.rs
+3 -3
View File
@@ -273,7 +273,7 @@ async fn handle_propfind(
let mut response_body = Vec::new();
CardDavAdapter::generate_contacts_response(
&mut response_body,
&[contact.clone()],
std::slice::from_ref(contact),
&[(contact.uid.clone(), contact_to_vcard(contact))],
&report,
base_href,
@@ -489,8 +489,8 @@ async fn handle_put(
fn extract_uid_from_vcard(vcard_data: &str) -> Option<String> {
for line in vcard_data.lines() {
let trimmed = line.trim();
if trimmed.starts_with("UID:") {
return Some(trimmed[4..].trim().to_string());
if let Some(stripped) = trimmed.strip_prefix("UID:") {
return Some(stripped.trim().to_string());
}
}
None
src/interfaces/api/handlers/chunked_upload_handler.rs
+6 -22
View File
@@ -103,8 +103,8 @@ impl ChunkedUploadHandler {
}
// ── Quota enforcement ────────────────────────────────────
if let Some(storage_svc) = state.storage_usage_service.as_ref() {
if let Err(err) = storage_svc
if let Some(storage_svc) = state.storage_usage_service.as_ref()
&& let Err(err) = storage_svc
.check_storage_quota(&auth_user.id, request.total_size)
.await
{
@@ -124,7 +124,6 @@ impl ChunkedUploadHandler {
)
.into_response();
}
}
// Validate chunk size if provided
let chunk_size = request.chunk_size.unwrap_or(DEFAULT_CHUNK_SIZE);
@@ -279,8 +278,8 @@ impl ChunkedUploadHandler {
let chunked_service = &state.core.chunked_upload_service;
let upload_service = &state.applications.file_upload_service;
// Assemble chunks
let (assembled_path, filename, folder_id, content_type, total_size) =
// Assemble chunks (hash-on-write: SHA-256 computed during assembly)
let (assembled_path, filename, folder_id, content_type, total_size, hash) =
match chunked_service.complete_upload(&upload_id).await {
Ok(result) => result,
Err(e) => {
@@ -300,24 +299,9 @@ impl ChunkedUploadHandler {
}
};
// Read assembled file and create final file record
let file_data = match tokio::fs::read(&assembled_path).await {
Ok(data) => data,
Err(e) => {
tracing::error!("Failed to read assembled file: {}", e);
return (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
"error": format!("Failed to read assembled file: {}", e)
})),
)
.into_response();
}
};
// Upload via normal service (this handles path resolution, metadata, etc.)
// Upload from assembled file on disk — zero extra RAM copies, hash pre-computed
match upload_service
.upload_file(filename.clone(), folder_id.clone(), content_type, file_data)
.upload_file_from_path(filename.clone(), folder_id.clone(), content_type, &assembled_path, Some(hash))
.await
{
Ok(file) => {
src/interfaces/api/handlers/file_handler.rs
+105 -128
View File
@@ -35,18 +35,22 @@ impl FileHandler {
// UPLOAD
// ═══════════════════════════════════════════════════════════════════════
/// Uploads a file with TRUE STREAMING support and Write-Behind Cache
/// Streaming file upload — constant ~64 KB RAM regardless of file size.
///
/// The three-tier strategy (write-behind / buffered / streaming) and dedup
/// are fully handled by `FileUploadUseCase::smart_upload`.
/// This handler only extracts multipart fields and maps the result to HTTP.
/// **Hash-on-Write**: SHA-256 is computed while spooling the multipart
/// body to the temp file. This eliminates the second sequential read
/// that dedup_service would otherwise need, cutting total I/O in half.
pub async fn upload_file(
State(state): State<GlobalState>,
auth_user: AuthUser,
mut multipart: Multipart,
) -> impl IntoResponse {
use sha2::{Digest, Sha256};
let upload_service = &state.applications.file_upload_service;
let mut folder_id: Option<String> = None;
tracing::debug!("📤 Processing file upload request");
tracing::debug!("📤 Processing streaming file upload (hash-on-write)");
while let Some(field) = multipart.next_field().await.unwrap_or(None) {
let name = field.name().unwrap_or("").to_string();
@@ -66,18 +70,82 @@ impl FileHandler {
.unwrap_or("application/octet-stream")
.to_string();
// Collect chunks from multipart
let mut chunks: Vec<Bytes> = Vec::new();
let mut total_size: usize = 0;
let mut field = field;
while let Ok(Some(chunk)) = field.chunk().await {
total_size += chunk.len();
chunks.push(chunk);
// ── Early quota check (before spooling to disk) ──────
// Use the multipart field's Content-Length header if present.
// If the user is already over quota, reject immediately
// without wasting I/O on spooling the entire body.
if let Some(storage_svc) = state.storage_usage_service.as_ref() {
let estimated_size = field
.headers()
.get(header::CONTENT_LENGTH)
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0);
if let Err(err) = storage_svc
.check_storage_quota(&auth_user.id, estimated_size)
.await
{
tracing::warn!(
"⛔ UPLOAD REJECTED (early quota): user={}, file={}, est_size={}",
auth_user.username,
filename,
estimated_size
);
return Self::quota_error_response(err).into_response();
}
}
// Empty file
if chunks.is_empty() {
let upload_service = &state.applications.file_upload_service;
// ── Spool multipart field to temp file + hash-on-write ──
let temp_dir = state.core.path_service.get_root_path().join(".dedup_temp");
let _ = tokio::fs::create_dir_all(&temp_dir).await;
let temp_path = temp_dir.join(format!("upload-{}", uuid::Uuid::new_v4()));
let mut total_size: u64 = 0;
let mut hasher = Sha256::new();
let spool_result: Result<(), String> = async {
let file = tokio::fs::File::create(&temp_path)
.await
.map_err(|e| format!("Failed to create temp file: {}", e))?;
// Pre-allocate if Content-Length is known (reduces fragmentation)
let hint = field
.headers()
.get(axum::http::header::CONTENT_LENGTH)
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u64>().ok());
if let Some(len) = hint {
let _ = file.set_len(len).await; // best-effort
}
// 512 KB buffer — 8× fewer write syscalls than 64 KB
let mut writer = tokio::io::BufWriter::with_capacity(524_288, file);
let mut field = field;
while let Ok(Some(chunk)) = field.chunk().await {
total_size += chunk.len() as u64;
hasher.update(&chunk);
tokio::io::AsyncWriteExt::write_all(&mut writer, &chunk)
.await
.map_err(|e| format!("Failed to write chunk: {}", e))?;
}
tokio::io::AsyncWriteExt::flush(&mut writer)
.await
.map_err(|e| format!("Failed to flush temp file: {}", e))?;
Ok(())
}
.await;
if let Err(e) = spool_result {
let _ = tokio::fs::remove_file(&temp_path).await;
tracing::error!("❌ UPLOAD SPOOL FAILED: {} - {}", filename, e);
return Self::domain_error_response(
crate::common::errors::DomainError::internal_error("FileUpload", e),
)
.into_response();
}
// Empty file — use in-memory path
if total_size == 0 {
let _ = tokio::fs::remove_file(&temp_path).await;
return match upload_service
.upload_file(filename, folder_id, content_type, vec![])
.await
@@ -87,127 +155,49 @@ impl FileHandler {
};
}
// Delegate to FileService (simple path, no write-behind/dedup)
let upload_service = &state.applications.file_upload_service;
let data = Self::combine_chunks(chunks, total_size);
match upload_service
.upload_file(filename.clone(), folder_id, content_type, data)
.await
{
Ok(file) => {
tracing::info!("✅ UPLOAD COMPLETE: {} (ID: {})", filename, file.id);
return Self::created_json_response(&file);
}
Err(err) => {
tracing::error!("❌ UPLOAD FAILED: {} - {}", filename, err);
return Self::domain_error_response(err);
}
}
}
}
(
StatusCode::BAD_REQUEST,
Json(serde_json::json!({
"error": "No file provided"
})),
)
.into_response()
}
/// Uploads a file with Write-Behind Cache + Dedup (smart strategy).
///
/// Delegates entirely to `FileUploadUseCase::smart_upload` which picks the
/// optimal tier and handles deduplication internally.
pub async fn upload_file_with_cache(
State(state): State<GlobalState>,
auth_user: AuthUser,
mut multipart: Multipart,
) -> impl IntoResponse {
let upload_service = &state.applications.file_upload_service;
let mut folder_id: Option<String> = None;
tracing::debug!("📤 Processing file upload request (with smart upload)");
while let Some(field) = multipart.next_field().await.unwrap_or(None) {
let name = field.name().unwrap_or("").to_string();
if name == "folder_id" {
let v = field.text().await.unwrap_or_default();
if !v.is_empty() {
folder_id = Some(v);
}
continue;
}
if name == "file" {
let filename = field.file_name().unwrap_or("unnamed").to_string();
let content_type = field
.content_type()
.unwrap_or("application/octet-stream")
.to_string();
// Collect chunks
let mut chunks: Vec<Bytes> = Vec::new();
let mut total_size: usize = 0;
let mut field = field;
while let Ok(Some(chunk)) = field.chunk().await {
total_size += chunk.len();
chunks.push(chunk);
}
// Empty file
if chunks.is_empty() {
let upload_svc = &state.applications.file_upload_service;
return match upload_svc
.upload_file(filename, folder_id, content_type, vec![])
.await
{
Ok(file) => Self::created_json_response(&file).into_response(),
Err(err) => Self::domain_error_response(err).into_response(),
};
}
// ── Quota enforcement ────────────────────────────────────
if let Some(storage_svc) = state.storage_usage_service.as_ref() {
if let Err(err) = storage_svc
.check_storage_quota(&auth_user.id, total_size as u64)
// Finalize hash
let hash = hex::encode(hasher.finalize());
// ── Quota enforcement ────────────────────────────────
if let Some(storage_svc) = state.storage_usage_service.as_ref()
&& let Err(err) = storage_svc
.check_storage_quota(&auth_user.id, total_size)
.await
{
let _ = tokio::fs::remove_file(&temp_path).await;
tracing::warn!(
"⛔ UPLOAD REJECTED (quota): user={}, file={}, size={} — {}",
"⛔ UPLOAD REJECTED (quota): user={}, file={}, size={}",
auth_user.username,
filename,
total_size,
err
total_size
);
return Self::quota_error_response(err).into_response();
}
}
// Delegate to smart_upload (handles write-behind, dedup, streaming)
// ── Streaming upload (temp file → blob store, hash pre-computed) ─
match upload_service
.smart_upload(
.upload_file_streaming(
filename.clone(),
folder_id,
content_type,
chunks,
&temp_path,
total_size,
Some(hash),
)
.await
{
Ok((file, strategy)) => {
Ok(file) => {
tracing::info!(
"✅ SMART UPLOAD: {} ({} bytes, strategy: {:?}, ID: {})",
"✅ STREAMING UPLOAD: {} ({} bytes, ID: {})",
filename,
total_size,
strategy,
file.id
);
return Self::created_json_response(&file).into_response();
}
Err(err) => {
tracing::error!("❌ SMART UPLOAD FAILED: {} - {}", filename, err);
let _ = tokio::fs::remove_file(&temp_path).await;
tracing::error!("❌ UPLOAD FAILED: {} - {}", filename, err);
return Self::domain_error_response(err).into_response();
}
}
@@ -451,7 +441,7 @@ impl FileHandler {
.is_some_and(|v| v == "true" || v == "1");
match retrieval
.get_file_optimized(&id, accept_webp, prefer_original)
.get_file_optimized_preloaded(&id, file_dto.clone(), accept_webp, prefer_original)
.await
{
Ok((_file, content)) => match content {
@@ -545,16 +535,16 @@ impl FileHandler {
/// Uploads a file and generates thumbnails in the background for images.
///
/// Delegates to [`Self::upload_file_with_cache`] and, on success, spawns
/// Delegates to [`Self::upload_file`] (streaming) and, on success, spawns
/// a background task to generate all thumbnail sizes.
pub async fn upload_file_with_thumbnails(
State(state): State<GlobalState>,
auth_user: AuthUser,
multipart: Multipart,
) -> impl IntoResponse {
// Use the smart upload handler
// Use the streaming upload handler
let response =
Self::upload_file_with_cache(State(state.clone()), auth_user, multipart).await;
Self::upload_file(State(state.clone()), auth_user, multipart).await;
// Try to extract file info for thumbnail generation
if let Ok(body_bytes) =
@@ -803,19 +793,6 @@ impl FileHandler {
// PRIVATE HELPERS
// ═══════════════════════════════════════════════════════════════════════
/// Combine chunks into a single Vec<u8>.
fn combine_chunks(chunks: Vec<Bytes>, total_size: usize) -> Vec<u8> {
if chunks.len() == 1 {
chunks.into_iter().next().unwrap().to_vec()
} else {
let mut combined = Vec::with_capacity(total_size);
for chunk in chunks {
combined.extend_from_slice(&chunk);
}
combined
}
}
/// Build a Content-Disposition header value.
fn content_disposition(name: &str, mime: &str, params: &HashMap<String, String>) -> String {
let force_inline = params
src/interfaces/api/handlers/i18n_handler.rs
+2 -2
View File
@@ -32,7 +32,7 @@ impl I18nHandler {
Query(query): Query<TranslationRequestDto>,
) -> impl IntoResponse {
let locale = match &query.locale {
Some(locale_str) => match Locale::from_str(locale_str) {
Some(locale_str) => match Locale::from_code(locale_str) {
Some(locale) => Some(locale),
None => {
let error = TranslationErrorDto {
@@ -86,7 +86,7 @@ impl I18nHandler {
State(_service): State<AppState>,
locale_code: String,
) -> impl IntoResponse {
let locale = match Locale::from_str(&locale_code) {
let locale = match Locale::from_code(&locale_code) {
Some(locale) => locale,
None => {
return (
src/interfaces/api/handlers/webdav_handler.rs
+25 -29
View File
@@ -404,17 +404,17 @@ async fn handle_get(
.await
.map_err(|_e| AppError::not_found(format!("File not found: {}", path)))?;
// Get file content
let content = file_retrieval_service
.get_file_content(&file.id)
// Stream file content — constant ~64 KB memory regardless of file size
let stream = file_retrieval_service
.get_file_stream(&file.id)
.await
.map_err(|e| AppError::internal_error(format!("Failed to get file content: {}", e)))?;
.map_err(|e| AppError::internal_error(format!("Failed to stream file: {}", e)))?;
// Build response
// Build streaming response using Content-Length from metadata
Ok(Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, file.mime_type)
.header(header::CONTENT_LENGTH, content.len())
.header(header::CONTENT_LENGTH, file.size)
.header(header::ETAG, format!("\"{}\"", file.id))
.header(
header::LAST_MODIFIED,
@@ -422,7 +422,7 @@ async fn handle_get(
.unwrap_or_else(Utc::now)
.to_rfc2822(),
)
.body(Body::from(content))
.body(Body::from_stream(Box::into_pin(stream)))
.unwrap())
}
@@ -458,21 +458,16 @@ async fn handle_head(
.unwrap());
}
// Try as file
// Try as file — use metadata only, never load content for HEAD
let file = file_retrieval_service
.get_file_by_path(&path)
.await
.map_err(|_e| AppError::not_found(format!("Resource not found: {}", path)))?;
let content = file_retrieval_service
.get_file_content(&file.id)
.await
.map_err(|e| AppError::internal_error(format!("Failed to get file content: {}", e)))?;
Ok(Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, &file.mime_type)
.header(header::CONTENT_LENGTH, content.len())
.header(header::CONTENT_LENGTH, file.size)
.header(header::ETAG, format!("\"{}\"", file.id))
.header(
header::LAST_MODIFIED,
@@ -960,32 +955,33 @@ async fn handle_copy(
}
}
} else {
// Try to copy file
// Copy file — use zero-copy dedup (only increments blob ref_count, no content loaded)
let file = file_retrieval_service
.get_file_by_path(&source_path)
.await
.map_err(|_e| AppError::not_found(format!("Resource not found: {}", source_path)))?;
// Get file content
let content = file_retrieval_service
.get_file_content(&file.id)
.await
.map_err(|e| AppError::internal_error(format!("Failed to get file content: {}", e)))?;
// Get destination parent path and filename
let dest_filename = destination_path
.split('/')
.next_back()
.unwrap_or(&destination_path);
// Get destination parent folder ID
let dest_parent_path = if let Some(idx) = destination_path.rfind('/') {
&destination_path[..idx]
} else {
""
};
// Create new file in destination
file_upload_service
.create_file(dest_parent_path, dest_filename, &content, &file.mime_type)
let target_folder_id = if dest_parent_path.is_empty() {
None
} else {
match folder_service.get_folder_by_path(dest_parent_path).await {
Ok(parent) => Some(parent.id),
Err(_) => None,
}
};
// Zero-copy: only creates a new metadata row + increments blob reference count.
// No file content is ever loaded into memory.
let file_management_service = &state.applications.file_management_service;
file_management_service
.copy_file(&file.id, target_folder_id)
.await
.map_err(|e| AppError::internal_error(format!("Failed to copy file: {}", e)))?;
}
src/interfaces/middleware/cache.rs
+6
View File
@@ -48,6 +48,12 @@ pub struct HttpCache {
default_max_age: u64,
}
impl Default for HttpCache {
fn default() -> Self {
Self::new()
}
}
impl HttpCache {
/// Creates a new cache instance
pub fn new() -> Self {
static/js/app.js
+59 -140
View File
@@ -3,6 +3,23 @@
* This file contains the core functionality, initialization and state management
*/
/**
* Escape HTML special characters to prevent XSS attacks.
* Use this whenever inserting user-provided text (file names, folder names, etc.) into HTML.
* @param {string} str - The string to escape
* @returns {string} The escaped string safe for HTML insertion
*/
function escapeHtml(str) {
if (typeof str !== 'string') return '';
return str
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#039;');
}
window.escapeHtml = escapeHtml;
// Global state
const app = {
currentView: 'grid', // Current view mode: 'grid' or 'list'
@@ -972,8 +989,8 @@ function addTrashItemToView(item) {
<div class="file-icon">
<i class="${iconClass}"></i>
</div>
<div class="file-name">${item.name}</div>
<div class="file-info">${typeLabel} - ${formattedDate}</div>
<div class="file-name">${escapeHtml(item.name)}</div>
<div class="file-info">${escapeHtml(typeLabel)} - ${escapeHtml(formattedDate)}</div>
<div class="trash-actions">
<button class="btn-restore" title="${window.i18n ? window.i18n.t('trash.restore') : 'Restore'}">
<i class="fas fa-undo"></i>
@@ -1013,11 +1030,11 @@ function addTrashItemToView(item) {
<div class="file-icon">
<i class="${iconClass}"></i>
</div>
<span>${item.name}</span>
<span>${escapeHtml(item.name)}</span>
</div>
<div class="type-cell">${typeLabel}</div>
<div class="path-cell">${item.original_path || '--'}</div>
<div class="date-cell">${formattedDate}</div>
<div class="type-cell">${escapeHtml(typeLabel)}</div>
<div class="path-cell">${escapeHtml(item.original_path || '--')}</div>
<div class="date-cell">${escapeHtml(formattedDate)}</div>
<div class="actions-cell">
<button class="btn-restore" title="${window.i18n ? window.i18n.t('trash.restore') : 'Restore'}">
<i class="fas fa-undo"></i>
@@ -1465,7 +1482,7 @@ async function refreshUserData() {
const token = localStorage.getItem(TOKEN_KEY);
console.log('refreshUserData called, token:', token ? token.substring(0, 20) + '...' : 'null');
if (!token || token === 'mock_token_emergency_bypass' || token === 'emergency_token') {
if (!token) {
console.log('No valid token, skipping user data refresh');
return null;
}
@@ -1579,92 +1596,12 @@ function showUserProfileModal() {
* Check if user is authenticated and load user's home folder
*/
async function checkAuthentication() {
// COMPLETE BREAK FOR AUTHENTICATION LOOPS:
// Always allow app to load with minimal authentication
// This is an emergency fix to stop the redirect loops
// Check URL for no_redirect parameter that indicates we should bypass auth
const bypassAuth = window.location.search.includes('no_redirect=true') ||
window.location.search.includes('bypass_auth=true');
if (bypassAuth) {
console.log('CRITICAL: Bypassing all authentication checks due to URL parameter');
// Always force a clean authentication state to break loops
const TOKEN_KEY = 'oxicloud_token';
const USER_DATA_KEY = 'oxicloud_user';
// Set a mock token if needed
if (!localStorage.getItem(TOKEN_KEY)) {
console.log('Setting mock token to prevent redirects');
localStorage.setItem(TOKEN_KEY, 'mock_token_emergency_bypass');
// Set expiry far in the future
localStorage.setItem('oxicloud_token_expiry',
new Date(Date.now() + 86400000 * 30).toISOString()); // 30 days
}
// Create minimal user data to make the app work
const userData = JSON.parse(localStorage.getItem(USER_DATA_KEY) || '{}');
if (!userData.username) {
console.log('No user data found, creating mock user');
const defaultUserData = {
id: 'default-user-id',
username: 'usuario',
email: 'usuario@example.com',
storage_quota_bytes: 10737418240, // 10GB default
storage_used_bytes: 0
};
localStorage.setItem(USER_DATA_KEY, JSON.stringify(defaultUserData));
// Update avatar with default initials
document.querySelectorAll('.user-avatar, .user-menu-avatar').forEach(el => el.textContent = 'US');
// Update storage display with default values
updateStorageUsageDisplay(defaultUserData);
} else {
// Update avatar with user initials
const userInitials = userData.username.substring(0, 2).toUpperCase();
document.querySelectorAll('.user-avatar, .user-menu-avatar').forEach(el => el.textContent = userInitials);
// Show cached storage first, then try to refresh from server
updateStorageUsageDisplay(userData);
// Try to get updated storage from server (if we have a real token)
const token = localStorage.getItem(TOKEN_KEY);
if (token && token !== 'mock_token_emergency_bypass' && token !== 'emergency_token') {
console.log('Bypass mode: Attempting to refresh storage from server...');
refreshUserData().then(freshData => {
if (freshData) {
console.log('Bypass mode: Storage updated from server');
}
}).catch(err => {
console.warn('Bypass mode: Could not refresh user data:', err);
});
}
}
// Reset all counters to prevent loops
sessionStorage.removeItem('redirect_count');
localStorage.setItem('refresh_attempts', '0');
// Proceed directly to load files
app.currentPath = '';
ui.updateBreadcrumb('');
loadFiles();
return;
}
try {
// Simplified authentication check - just verify token exists
const TOKEN_KEY = 'oxicloud_token';
const REFRESH_TOKEN_KEY = 'oxicloud_refresh_token';
const TOKEN_EXPIRY_KEY = 'oxicloud_token_expiry';
const USER_DATA_KEY = 'oxicloud_user';
// Reset counters to prevent loops
sessionStorage.removeItem('redirect_count');
localStorage.setItem('refresh_attempts', '0');
// --- OIDC exchange code handling ---
// After OIDC login, the backend redirects here with ?oidc_code=...
const urlParams = new URLSearchParams(window.location.search);
@@ -1737,18 +1674,16 @@ async function checkAuthentication() {
}
}
// Simple token check - just verify it exists
// Verify token exists
const token = localStorage.getItem(TOKEN_KEY);
if (!token) {
console.log('No token found, redirecting to login');
// Avoid potential loop by adding a parameter
const redirectUrl = '/login?source=app';
window.location.href = redirectUrl;
window.location.href = '/login?source=app';
return;
}
// Token exists, proceed with minimal validation
// Token exists, proceed with app initialization
console.log('Token found, proceeding with app initialization');
// Display user information if available
@@ -1769,7 +1704,6 @@ async function checkAuthentication() {
updateStorageUsageDisplay(userData);
// Then refresh user data from server in the background to get updated storage
// This triggers the backend to recalculate storage and returns fresh data
refreshUserData().then(freshData => {
if (freshData) {
console.log('Storage usage updated from server');
@@ -1781,56 +1715,41 @@ async function checkAuthentication() {
// Find and load the user's home folder
findUserHomeFolder(userData.username);
} else {
// If no user data but we have a token, create default user data
console.log('No user data but token exists, using default user');
const defaultUserData = {
id: 'default-user-id',
username: 'usuario',
email: 'usuario@example.com',
storage_quota_bytes: 10737418240, // 10GB default
storage_used_bytes: 0
};
localStorage.setItem(USER_DATA_KEY, JSON.stringify(defaultUserData));
// Update avatar with default initials
document.querySelectorAll('.user-avatar, .user-menu-avatar').forEach(el => el.textContent = 'US');
// Update storage display with default values
updateStorageUsageDisplay(defaultUserData);
// Find and load default folder
app.currentPath = '';
ui.updateBreadcrumb('');
loadFiles();
// No user data but token exists — try to fetch from server
console.log('No user data, attempting to fetch from server');
try {
const freshData = await refreshUserData();
if (freshData && freshData.username) {
const userInitials = freshData.username.substring(0, 2).toUpperCase();
document.querySelectorAll('.user-avatar, .user-menu-avatar').forEach(el => el.textContent = userInitials);
updateStorageUsageDisplay(freshData);
findUserHomeFolder(freshData.username);
} else {
// Server didn't return valid user data — token is likely invalid
console.warn('Could not retrieve user data, redirecting to login');
localStorage.removeItem(TOKEN_KEY);
localStorage.removeItem(REFRESH_TOKEN_KEY);
localStorage.removeItem(TOKEN_EXPIRY_KEY);
localStorage.removeItem(USER_DATA_KEY);
window.location.href = '/login?source=invalid_session';
}
} catch (err) {
console.error('Failed to fetch user data:', err);
localStorage.removeItem(TOKEN_KEY);
localStorage.removeItem(REFRESH_TOKEN_KEY);
localStorage.removeItem(TOKEN_EXPIRY_KEY);
localStorage.removeItem(USER_DATA_KEY);
window.location.href = '/login?source=session_error';
}
}
} catch (error) {
console.error('Error during authentication check:', error);
// CRITICAL: On any error, create emergency bypass to break any loops
console.log('Creating emergency authentication bypass due to error');
localStorage.setItem('oxicloud_token', 'emergency_token');
localStorage.setItem('oxicloud_token_expiry',
new Date(Date.now() + 86400000 * 30).toISOString()); // 30 days
const defaultUserData = {
id: 'emergency-user-id',
username: 'usuario',
email: 'usuario@example.com',
storage_quota_bytes: 10737418240, // 10GB default
storage_used_bytes: 0
};
localStorage.setItem('oxicloud_user', JSON.stringify(defaultUserData));
// Update avatar
document.querySelectorAll('.user-avatar, .user-menu-avatar').forEach(el => el.textContent = 'US');
// Update storage display with default values
updateStorageUsageDisplay(defaultUserData);
// Load root files
app.currentPath = '';
ui.updateBreadcrumb('');
loadFiles();
// On error, redirect to login cleanly — never create fake tokens
localStorage.removeItem('oxicloud_token');
localStorage.removeItem('oxicloud_refresh_token');
localStorage.removeItem('oxicloud_token_expiry');
localStorage.removeItem('oxicloud_user');
window.location.href = '/login?source=auth_error';
}
}
static/js/auth.js
+37 -142
View File
@@ -513,8 +513,10 @@ let authInitialized = false;
const timeSinceCleanup = Date.now() - lastCleanup;
if (lastCleanup > 0 && timeSinceCleanup < 10000) { // Less than 10 seconds
console.warn('Multiple auth problems in short time, enabling direct bypass mode');
localStorage.setItem('bypass_auth_mode', 'true');
console.warn('Multiple auth problems in short time, clearing auth data');
localStorage.removeItem('oxicloud_token');
localStorage.removeItem('oxicloud_refresh_token');
localStorage.removeItem('oxicloud_token_expiry');
}
})();
@@ -577,7 +579,7 @@ document.addEventListener('DOMContentLoaded', () => {
}
// Redirect to main app
window.location.href = '/?no_redirect=true';
window.location.href = '/';
return;
}
} catch (err) {
@@ -690,8 +692,12 @@ if (isLoginPage && loginForm) {
console.log("Login response:", data); // Log the response for debugging
// Use the correct field names from our API response
const token = data.access_token || data.token || "mock_access_token";
const refreshToken = data.refresh_token || data.refreshToken || "mock_refresh_token";
const token = data.access_token || data.token;
const refreshToken = data.refresh_token || data.refreshToken;
if (!token) {
throw new Error('Server did not return an access token');
}
localStorage.setItem(TOKEN_KEY, token);
localStorage.setItem(REFRESH_TOKEN_KEY, refreshToken);
@@ -731,20 +737,13 @@ if (isLoginPage && loginForm) {
// Reset redirect counter on successful login
sessionStorage.removeItem('redirect_count');
// Fetch and store user data
// Use the user data directly from the response
const userData = data.user || {
id: 'test-user-id',
username: username,
email: username + '@example.com',
role: 'user',
active: true,
storage_quota_bytes: 10737418240, // 10GB default
storage_used_bytes: 0
};
console.log("Storing user data:", userData);
localStorage.setItem(USER_DATA_KEY, JSON.stringify(userData));
// Fetch and store user data from the response
if (data.user) {
console.log("Storing user data from server");
localStorage.setItem(USER_DATA_KEY, JSON.stringify(data.user));
} else {
console.warn("Server did not return user data — will fetch from /me endpoint after redirect");
}
// Redirect to main app
redirectToMainApp();
@@ -859,25 +858,6 @@ async function login(username, password) {
try {
console.log(`Attempting to login with username: ${username}`);
// Special case for test user
if (username === 'test' && password === 'test') {
console.log('Using test user fallback');
// Return a mock response that matches our backend structure
return {
user: {
id: "test-user-id",
username: "test",
email: "test@example.com",
role: "user",
active: true
},
access_token: "mock_access_token",
refresh_token: "mock_refresh_token",
token_type: "Bearer",
expires_in: 3600
};
}
// Add better error handling with timeout
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout
@@ -928,19 +908,6 @@ async function register(username, email, password, role = 'user') {
try {
console.log(`Attempting to register user: ${username}`);
// Special case for test user
if (username === 'test') {
console.log('Using test user registration fallback');
// Return a mock user response
return {
id: "test-user-id",
username: username,
email: email,
role: role || "user",
active: true
};
}
const response = await fetch(REGISTER_ENDPOINT, {
method: 'POST',
headers: {
@@ -1005,7 +972,6 @@ async function fetchUserData(token) {
*/
async function refreshAuthToken(refreshToken) {
try {
console.log("CRITICAL: Token refresh disabled to prevent infinite loop");
// Check if we're in a refresh loop
const refreshAttempts = parseInt(localStorage.getItem('refresh_attempts') || '0');
localStorage.setItem('refresh_attempts', (refreshAttempts + 1).toString());
@@ -1021,49 +987,15 @@ async function refreshAuthToken(refreshToken) {
throw new Error('Too many refresh attempts, forcing login');
}
// For test users, generate a fake response that will work
// This ensures the app works with test accounts
const isMockToken = refreshToken === "mock_refresh_token" || refreshToken.includes("mock");
if (isMockToken) {
console.log("Using mock refresh token response");
// Create a simulated token with no expiration
const timestamp = Math.floor(Date.now() / 1000);
const expiry = timestamp + 86400 * 30; // 30 days
// Create a basic token with a very long expiry
const mockUserData = {
id: "default-user-id",
username: "usuario",
email: "usuario@example.com",
role: "user",
active: true
};
// Store directly in localStorage to bypass token parsing
localStorage.setItem(USER_DATA_KEY, JSON.stringify(mockUserData));
localStorage.setItem(TOKEN_KEY, "mock_token_preventing_loops");
localStorage.setItem(TOKEN_EXPIRY_KEY, new Date(expiry * 1000).toISOString());
// Reset counters
sessionStorage.removeItem('redirect_count');
localStorage.setItem('refresh_attempts', '0');
return {
user: mockUserData,
access_token: "mock_token_preventing_loops",
refresh_token: "mock_refresh_token_new",
token_type: "Bearer",
expires_in: 86400 * 30
};
if (!refreshToken) {
throw new Error('No refresh token available');
}
// If it's not a mock token, let's try the normal refresh but with extra safeguards
console.log("Attempting to refresh real token with safety limits");
console.log("Attempting to refresh token");
// Extra timeout for safety
// Timeout for safety
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 3000); // Reduced to 3 second timeout
const timeoutId = setTimeout(() => controller.abort(), 5000);
const response = await fetch(REFRESH_ENDPOINT, {
method: 'POST',
@@ -1138,75 +1070,38 @@ async function checkFirstRun() {
/**
* Redirect to main application
* Complete rewrite with multiple failsafes to prevent redirect loops
*/
function redirectToMainApp() {
console.log('Redirecting to main application with anti-loop measures');
console.log('Redirecting to main application');
try {
// Check if we're in bypass mode
const bypassMode = localStorage.getItem('bypass_auth_mode') === 'true';
// Calculate which URL parameter to use
let param = 'no_redirect=true';
// Add strong bypass parameter if in bypass mode
if (bypassMode) {
param = 'bypass_auth=true';
console.log('CRITICAL: Using emergency bypass mode for redirection');
}
// Reset refresh attempts counter on redirection
localStorage.setItem('refresh_attempts', '0');
sessionStorage.removeItem('redirect_count');
// Set a token expiry if none exists (to prevent potential loops)
// Set a token expiry if none exists
const tokenExpiry = localStorage.getItem(TOKEN_EXPIRY_KEY);
if (!tokenExpiry) {
console.log('Setting default token expiry before redirect');
const expiryTime = new Date();
expiryTime.setDate(expiryTime.getDate() + 30); // 30 days
expiryTime.setDate(expiryTime.getDate() + 30);
localStorage.setItem(TOKEN_EXPIRY_KEY, expiryTime.toISOString());
}
// Additional guard: ensure we have at least some form of token
// Verify we have a valid token before redirecting
const hasToken = localStorage.getItem(TOKEN_KEY);
if (!hasToken && !bypassMode) {
console.warn('No token found before redirect, creating emergency token');
localStorage.setItem(TOKEN_KEY, 'emergency_redirect_token');
}
// Log that we're about to redirect
console.log(`Redirecting to app with param: ${param}`);
// Use a timeout to prevent any potential race conditions
setTimeout(() => {
try {
// Navigate to the main app with the appropriate parameter
window.location.replace(`/?${param}`);
} catch (innerError) {
console.error('Critical error during redirection:', innerError);
// Ultimate fallback - clear everything and go to a special error page
localStorage.clear();
sessionStorage.clear();
window.location.href = '/login?critical=redirect_error';
}
}, 50);
} catch (error) {
console.error('Fatal error in redirectToMainApp:', error);
// Emergency fallback
try {
window.location.href = '/login?error=redirect_fatal';
} catch (e) {
// Nothing more we can do
alert('Critical redirect error. Please reload the page and try again.');
}
}
// No more redirect checks or token validation
if (!hasToken) {
console.error('No token found, cannot redirect to app');
return;
}
// Navigate to the main app
window.location.replace('/');
} catch (error) {
console.error('Error during redirect:', error);
window.location.href = '/login?error=redirect_failed';
}
}
/**
* Logout - clear tokens and redirect to login
*/
static/js/contextMenus.js
+1 -1
View File
@@ -424,7 +424,7 @@ const contextMenus = {
const folderItem = document.createElement('div');
folderItem.className = 'folder-select-item';
folderItem.dataset.folderId = folder.id;
folderItem.innerHTML = `<i class="fas fa-folder"></i> ${folder.name}`;
folderItem.innerHTML = `<i class="fas fa-folder"></i> ${escapeHtml(folder.name)}`;
folderItem.addEventListener('click', () => {
// Deselect all
static/js/favorites.js
+5 -5
View File
@@ -501,7 +501,7 @@ const favorites = {
<div class="file-icon folder-icon">
<i class="fas fa-folder"></i>
</div>
<div class="file-name">${folder.name}</div>
<div class="file-name">${escapeHtml(folder.name)}</div>
<div class="file-info">Folder</div>
`;
@@ -550,7 +550,7 @@ const favorites = {
<div class="file-icon folder-icon">
<i class="fas fa-folder"></i>
</div>
<span>${folder.name}</span>
<span>${escapeHtml(folder.name)}</span>
</div>
<div class="type-cell">${window.i18n ? window.i18n.t('files.file_types.folder') : 'Folder'}</div>
<div class="size-cell">--</div>
@@ -639,7 +639,7 @@ const favorites = {
<div class="file-icon ${iconSpecialClass}">
<i class="${iconClass}"></i>
</div>
<div class="file-name">${file.name}</div>
<div class="file-name">${escapeHtml(file.name)}</div>
<div class="file-info">Modified ${formattedDate.split(' ')[0]}</div>
`;
@@ -685,9 +685,9 @@ const favorites = {
<div class="file-icon ${iconSpecialClass}">
<i class="${iconClass}"></i>
</div>
<span>${file.name}</span>
<span>${escapeHtml(file.name)}</span>
</div>
<div class="type-cell">${typeLabel}</div>
<div class="type-cell">${escapeHtml(typeLabel)}</div>
<div class="size-cell">${fileSize}</div>
<div class="date-cell">${formattedDate}</div>
`;
static/js/fileRenderer.js
+4 -4
View File
@@ -220,7 +220,7 @@ class FileRenderer {
<div class="file-icon folder-icon">
<i class="fas fa-folder"></i>
</div>
<div class="file-name">${item.name}</div>
<div class="file-name">${window.escapeHtml(item.name)}</div>
`;
// Make draggable
@@ -274,7 +274,7 @@ class FileRenderer {
<div class="file-icon">
<i class="${iconClass}"></i>
</div>
<div class="file-name">${item.name}</div>
<div class="file-name">${window.escapeHtml(item.name)}</div>
`;
// Make draggable
@@ -343,7 +343,7 @@ class FileRenderer {
<div class="file-icon folder-icon">
<i class="fas fa-folder"></i>
</div>
<span>${item.name}</span>
<span>${window.escapeHtml(item.name)}</span>
</div>
<div>${this.i18n.t('files.file_types.folder')}</div>
<div>--</div>
@@ -416,7 +416,7 @@ class FileRenderer {
<div class="file-icon">
<i class="${iconClass}"></i>
</div>
<span>${item.name}</span>
<span>${window.escapeHtml(item.name)}</span>
</div>
<div>${typeLabel}</div>
<div>${fileSize}</div>
static/js/recent.js
+3 -3
View File
@@ -246,7 +246,7 @@ const recent = {
<div class="file-icon ${iconSpecialClass}">
<i class="${iconClass}"></i>
</div>
<div class="file-name">${file.name}</div>
<div class="file-name">${escapeHtml(file.name)}</div>
<div class="file-info">Accessed ${formattedDate.split(' ')[0]}</div>
`;
@@ -297,9 +297,9 @@ const recent = {
<div class="file-icon ${iconSpecialClass}">
<i class="${iconClass}"></i>
</div>
<span>${file.name}</span>
<span>${escapeHtml(file.name)}</span>
</div>
<div class="type-cell">${typeLabel}</div>
<div class="type-cell">${escapeHtml(typeLabel)}</div>
<div class="size-cell">${fileSize}</div>
<div class="date-cell">${formattedDate}</div>
`;
static/js/ui.js
+4 -4
View File
@@ -677,7 +677,7 @@ const ui = {
<div class="file-icon folder-icon">
<i class="fas fa-folder"></i>
</div>
<div class="file-name">${folder.name}</div>
<div class="file-name">${escapeHtml(folder.name)}</div>
<div class="file-info">Folder</div>
`;
@@ -818,7 +818,7 @@ const ui = {
<div class="file-icon folder-icon">
<i class="fas fa-folder"></i>
</div>
<span>${folder.name}</span>
<span>${escapeHtml(folder.name)}</span>
${isFolderFav ? '<i class="fas fa-star favorite-star-inline"></i>' : ''}
</div>
<div class="type-cell">${window.i18n ? window.i18n.t('files.file_types.folder') : 'Folder'}</div>
@@ -952,7 +952,7 @@ const ui = {
<div class="file-icon">
<i class="${iconClass}"></i>
</div>
<div class="file-name">${file.name}</div>
<div class="file-name">${escapeHtml(file.name)}</div>
<div class="file-info">Modified ${formattedDate.split(' ')[0]}</div>
`;
@@ -1056,7 +1056,7 @@ const ui = {
<div class="file-icon ${iconSpecialClass}">
<i class="${iconClass}"></i>
</div>
<span>${file.name}</span>
<span>${escapeHtml(file.name)}</span>
${isFileFav ? '<i class="fas fa-star favorite-star-inline"></i>' : ''}
</div>
<div class="type-cell">${typeLabel}</div>
storage/.trash/trash_index.json
-1
View File
@@ -1 +0,0 @@
[]
storage/file_ids.json
-5
View File
@@ -1,5 +0,0 @@
{
"path_to_id": {},
"id_to_path": {},
"version": 0
}
storage/folder_ids.json
-19
View File
@@ -1,19 +0,0 @@
{
"path_to_id": {
"/Mi Carpeta - testuser999": "7c1afffd-867f-4649-a0f6-4cc8a9979af4",
"/Mi Carpeta - test": "edfc23e0-d9f9-4cc7-b1dc-6369568b19e6",
"/Mi Carpeta - user1": "b775c641-a698-458b-a561-42ae93073c74",
"/Mi Carpeta - testadmin": "02aed53b-f194-4e28-8680-2cc81a3584b3",
"/Mi Carpeta - testuser": "304384d6-9a7e-4b22-a659-287957e8b667",
"/Mi Carpeta - admin": "562e90aa-a3fc-4672-a0fd-9187fef5d4f1"
},
"id_to_path": {
"562e90aa-a3fc-4672-a0fd-9187fef5d4f1": "/Mi Carpeta - admin",
"7c1afffd-867f-4649-a0f6-4cc8a9979af4": "/Mi Carpeta - testuser999",
"02aed53b-f194-4e28-8680-2cc81a3584b3": "/Mi Carpeta - testadmin",
"304384d6-9a7e-4b22-a659-287957e8b667": "/Mi Carpeta - testuser",
"b775c641-a698-458b-a561-42ae93073c74": "/Mi Carpeta - user1",
"edfc23e0-d9f9-4cc7-b1dc-6369568b19e6": "/Mi Carpeta - test"
},
"version": 6
}