diff --git a/.gitignore b/.gitignore index 59fd19e4..d5d47b6c 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,9 @@ npm-debug.log *.log logs/ +# Storage data (user files, blobs — never commit) +storage/ + # Temporary files *.tmp *.bak diff --git a/Cargo.lock b/Cargo.lock index 363bbbd1..ffc29dd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,6 +88,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -422,6 +433,24 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -673,6 +702,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1584,6 +1623,26 @@ dependencies = [ "syn", ] +[[package]] +name = "moka" +version = "0.12.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac832c50ced444ef6be0767a008b02c106a909ba79d1d830501e94b96f6b7e" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + [[package]] name = "moxcms" version = "0.7.11" @@ -1716,6 +1775,7 @@ dependencies = [ "memmap2", "mime_guess", "mockall", + "moka", "quick-xml", "rand_core 0.6.4", "reqwest", @@ -1887,6 +1947,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + [[package]] name = "potential_utf" version = "0.1.4" @@ -2769,6 +2835,12 @@ dependencies = [ "syn", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tempfile" version = "3.25.0" diff --git a/Cargo.toml b/Cargo.toml index c9bfdd95..bfced7eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ hyper = { version = "1.8.1", features = ["full"] } quick-xml = "0.39.0" dotenvy = "0.15.7" lru = "0.16.3" +moka = { version = "0.12", features = ["future"] } memmap2 = "0.9" http-range-header = "0.4" image = { version = "0.25", default-features = false, features = ["jpeg", "png", "gif", "webp"] } diff --git a/bench.sh b/bench.sh new file mode 100755 index 00000000..55258511 --- /dev/null +++ b/bench.sh @@ -0,0 +1,419 @@ +#!/usr/bin/env bash +# ============================================================================ +# OxiCloud Performance Benchmark v3 +# Detailed RAM (RSS/VmPeak/VmSwap) + CPU monitoring + huge file downloads +# +# - Uploads: 1KB..500MB via API (above ~1GB the upload handler OOMs — +# that's a known issue in the multipart handler which buffers chunks). +# - Downloads: 1KB..10GB. For files >500MB we inject blobs directly into +# the blob store + DB so the streaming download path is tested without +# going through the buffering upload handler. +# ============================================================================ +set -uo pipefail + +BASE_URL="${BASE_URL:-http://127.0.0.1:8086}" +USERNAME="${BENCH_USER:-benchuser}" +PASSWORD="${BENCH_PASS:-BenchPass123!}" +FOLDER_ID="${BENCH_FOLDER:-0e72efc0-0d1c-45a1-b434-52336643b3f7}" +USER_ID="${BENCH_USER_ID:-c410b103-7b86-4ac2-9eb4-3804351547be}" +WORKDIR="/tmp/oxibench" +RESULTS="$WORKDIR/results_v3.csv" +STORAGE="./storage" +MONITOR_INTERVAL=0.2 # 200ms sample rate + +# ── Colours ────────────────────────────────────────────────────────────────── +RED='\033[0;31m'; GRN='\033[0;32m'; YEL='\033[1;33m' +CYA='\033[0;36m'; BLD='\033[1m'; DIM='\033[2m'; NC='\033[0m' + +header() { printf "\n${BLD}${CYA}═══════════════════════════════════════════${NC}\n"; \ + printf "${BLD}${CYA} %s${NC}\n" "$1"; \ + printf "${BLD}${CYA}═══════════════════════════════════════════${NC}\n"; } +ok() { printf "${GRN}✓${NC} %s\n" "$1"; } +warn() { printf "${YEL}⚠${NC} %s\n" "$1"; } +fail() { printf "${RED}✗${NC} %s\n" "$1"; } + +# ── Server PID ─────────────────────────────────────────────────────────────── +SERVER_PID="" +find_server_pid() { + SERVER_PID=$(pgrep -f "target/release/oxicloud" 2>/dev/null | head -1 || \ + pgrep -f "target/debug/oxicloud" 2>/dev/null | head -1 || echo "") + [[ -z "$SERVER_PID" ]] && { fail "Server PID not found"; exit 1; } +} + +# ── Memory (KB) ───────────────────────────────────────────────────────────── +get_rss_kb() { awk '/^VmRSS:/ {print $2}' /proc/$SERVER_PID/status 2>/dev/null || echo 0; } +get_peak_kb() { awk '/^VmPeak:/ {print $2}' /proc/$SERVER_PID/status 2>/dev/null || echo 0; } +get_swap_kb() { awk '/^VmSwap:/ {print $2}' /proc/$SERVER_PID/status 2>/dev/null || echo 0; } +get_vsize_kb() { awk '/^VmSize:/ {print $2}' /proc/$SERVER_PID/status 2>/dev/null || echo 0; } + +# ── CPU jiffies ────────────────────────────────────────────────────────────── +get_cpu_jiffies() { awk '{print $14+$15}' /proc/$SERVER_PID/stat 2>/dev/null || echo 0; } + +# ── Background monitor ────────────────────────────────────────────────────── +MONITOR_PID="" +MONITOR_RESULT="$WORKDIR/_mon" +MON_PEAK_RSS=0; MON_PEAK_VM=0; MON_PEAK_SWAP=0; MON_CPU_PCT="0.0" + +start_monitor() { + rm -f "$MONITOR_RESULT" + ( + peak_rss=0; peak_vm=0; peak_swap=0 + cpu0=$(get_cpu_jiffies); t0=$(date +%s%N) + while true; do + r=$(get_rss_kb); v=$(get_vsize_kb); s=$(get_swap_kb) + (( r > peak_rss )) && peak_rss=$r + (( v > peak_vm )) && peak_vm=$v + (( s > peak_swap )) && peak_swap=$s + sleep $MONITOR_INTERVAL 2>/dev/null || break + done + cpu1=$(get_cpu_jiffies); t1=$(date +%s%N) + dt=$(( cpu1 - cpu0 )); wall=$(( (t1 - t0) / 1000000 )) + cpup=0 + (( wall > 0 )) && cpup=$(echo "scale=1; $dt * 10 * 100 / $wall" | bc 2>/dev/null || echo 0) + echo "$peak_rss $peak_vm $peak_swap $cpup" > "$MONITOR_RESULT" + ) & + MONITOR_PID=$! +} + +stop_monitor() { + [[ -n "$MONITOR_PID" ]] && { kill $MONITOR_PID 2>/dev/null; wait $MONITOR_PID 2>/dev/null || true; MONITOR_PID=""; } + if [[ -f "$MONITOR_RESULT" ]]; then + read -r MON_PEAK_RSS MON_PEAK_VM MON_PEAK_SWAP MON_CPU_PCT < "$MONITOR_RESULT" + else + MON_PEAK_RSS=0; MON_PEAK_VM=0; MON_PEAK_SWAP=0; MON_CPU_PCT="0.0" + fi +} + +# ── Auth ───────────────────────────────────────────────────────────────────── +TOKEN="" +login() { + TOKEN=$(curl -sf -X POST "$BASE_URL/api/auth/login" \ + -H "Content-Type: application/json" \ + -d "{\"username\":\"$USERNAME\",\"password\":\"$PASSWORD\"}" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])" 2>/dev/null || echo "") + [[ -z "$TOKEN" ]] && { fail "JWT login failed"; exit 1; } +} + +# ── File gen ───────────────────────────────────────────────────────────────── +gen_file() { + local path="$1" bytes="$2" + local bs=1048576 + local count=$(( bytes / bs )) rem=$(( bytes % bs )) + (( count > 0 )) && dd if=/dev/urandom of="$path" bs=$bs count=$count 2>/dev/null + (( rem > 0 )) && dd if=/dev/urandom bs=$rem count=1 >> "$path" 2>/dev/null +} + +human() { + local b=$1 + if (( b >= 1073741824 )); then printf "%.2f GB" "$(echo "$b/1073741824" | bc -l)" + elif (( b >= 1048576 )); then printf "%.1f MB" "$(echo "$b/1048576" | bc -l)" + elif (( b >= 1024 )); then printf "%.1f KB" "$(echo "$b/1024" | bc -l)" + else printf "%d B" "$b"; fi +} + +throughput() { + local b=$1 ms=$2 + (( ms <= 0 )) && { echo "---"; return; } + human $(echo "$b*1000/$ms" | bc 2>/dev/null || echo 0) +} + +mb() { echo $(( $1 / 1024 )); } # KB -> MB + +# ── Inject large blob (bypass upload handler) ────────────────────────────── +# Creates a random file as a blob, registers it in DB, returns file_id. +# Usage: inject_blob