336 lines
12 KiB
Python
336 lines
12 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""
|
||
|
|
RustFS 旧桶清理脚本
|
||
|
|
|
||
|
|
删除所有遗留的旧桶结构，包括：
|
||
|
|
- moldinsight-geometry
|
||
|
|
- moldinsight-stp-files
|
||
|
|
- moldinsight-mold-cavities
|
||
|
|
- moldinsight-html
|
||
|
|
- moldinsight-user-files
|
||
|
|
|
||
|
|
注意：这是rustFS，而不是minio，只是用了minio的通用S3接口
|
||
|
|
"""
|
||
|
|
|
||
|
|
import asyncio
|
||
|
|
import sys
|
||
|
|
import os
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import List, Dict
|
||
|
|
from datetime import datetime
|
||
|
|
|
||
|
|
# 添加项目根目录和 src 目录到 Python 路径
|
||
|
|
project_root = Path(__file__).parent
|
||
|
|
src_root = project_root / "src"
|
||
|
|
sys.path.insert(0, str(project_root))
|
||
|
|
sys.path.insert(0, str(src_root))
|
||
|
|
|
||
|
|
from storage.rustfs_storage import RustFSManager
|
||
|
|
from config.settings import settings
|
||
|
|
from utils.logger import get_logger
|
||
|
|
|
||
|
|
logger = get_logger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
class RustFSCleanup:
|
||
|
|
"""RustFS 旧桶清理器"""
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
self.rustfs = RustFSManager()
|
||
|
|
self.is_connected = False
|
||
|
|
|
||
|
|
# 所有需要清理的旧桶
|
||
|
|
self.old_buckets_to_clean = [
|
||
|
|
'moldinsight-geometry',
|
||
|
|
'moldinsight-stp-files',
|
||
|
|
'moldinsight-mold-cavities',
|
||
|
|
'moldinsight-html',
|
||
|
|
'moldinsight-user-files'
|
||
|
|
]
|
||
|
|
|
||
|
|
async def connect(self):
|
||
|
|
"""连接到 RustFS"""
|
||
|
|
try:
|
||
|
|
await self.rustfs.connect(
|
||
|
|
endpoint=settings.RUSTFS_ENDPOINT,
|
||
|
|
access_key=settings.RUSTFS_ACCESS_KEY,
|
||
|
|
secret_key=settings.RUSTFS_SECRET_KEY,
|
||
|
|
timeout=settings.RUSTFS_TIMEOUT
|
||
|
|
)
|
||
|
|
self.is_connected = True
|
||
|
|
logger.info("RustFS 连接成功")
|
||
|
|
return True
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"RustFS 连接失败: {e}")
|
||
|
|
return False
|
||
|
|
|
||
|
|
async def close(self):
|
||
|
|
"""关闭连接"""
|
||
|
|
await self.rustfs.close()
|
||
|
|
self.is_connected = False
|
||
|
|
logger.info("RustFS 连接已关闭")
|
||
|
|
|
||
|
|
async def list_all_buckets(self) -> List[str]:
|
||
|
|
"""列出所有桶"""
|
||
|
|
if not self.is_connected:
|
||
|
|
raise RuntimeError("RustFS 未连接")
|
||
|
|
|
||
|
|
try:
|
||
|
|
buckets = self.rustfs.client.list_buckets()
|
||
|
|
bucket_names = [bucket.name for bucket in buckets]
|
||
|
|
logger.info(f"当前存在的桶: {bucket_names}")
|
||
|
|
return bucket_names
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"列出桶失败: {e}")
|
||
|
|
return []
|
||
|
|
|
||
|
|
async def check_bucket_status(self) -> Dict[str, Dict]:
|
||
|
|
"""检查桶状态和文件数量"""
|
||
|
|
if not self.is_connected:
|
||
|
|
raise RuntimeError("RustFS 未连接")
|
||
|
|
|
||
|
|
bucket_status = {}
|
||
|
|
|
||
|
|
for bucket_name in self.old_buckets_to_clean:
|
||
|
|
try:
|
||
|
|
# 检查桶是否存在
|
||
|
|
exists = self.rustfs.client.bucket_exists(bucket_name)
|
||
|
|
|
||
|
|
if exists:
|
||
|
|
# 统计文件数量
|
||
|
|
objects = list(self.rustfs.client.list_objects(bucket_name, recursive=True))
|
||
|
|
file_count = len(objects)
|
||
|
|
|
||
|
|
# 计算总大小
|
||
|
|
total_size = sum(obj.size for obj in objects)
|
||
|
|
|
||
|
|
bucket_status[bucket_name] = {
|
||
|
|
'exists': True,
|
||
|
|
'file_count': file_count,
|
||
|
|
'total_size': total_size,
|
||
|
|
'files': [obj.object_name for obj in objects]
|
||
|
|
}
|
||
|
|
|
||
|
|
logger.info(f"桶 {bucket_name}: 存在, {file_count} 个文件, {total_size} 字节")
|
||
|
|
else:
|
||
|
|
bucket_status[bucket_name] = {
|
||
|
|
'exists': False,
|
||
|
|
'file_count': 0,
|
||
|
|
'total_size': 0,
|
||
|
|
'files': []
|
||
|
|
}
|
||
|
|
logger.info(f"桶 {bucket_name}: 不存在")
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"检查桶 {bucket_name} 状态失败: {e}")
|
||
|
|
bucket_status[bucket_name] = {
|
||
|
|
'exists': False,
|
||
|
|
'file_count': 0,
|
||
|
|
'total_size': 0,
|
||
|
|
'files': [],
|
||
|
|
'error': str(e)
|
||
|
|
}
|
||
|
|
|
||
|
|
return bucket_status
|
||
|
|
|
||
|
|
async def delete_bucket(self, bucket_name: str) -> Dict[str, any]:
|
||
|
|
"""删除单个桶及其所有文件"""
|
||
|
|
result = {
|
||
|
|
'bucket_name': bucket_name,
|
||
|
|
'success': False,
|
||
|
|
'files_deleted': 0,
|
||
|
|
'error': None
|
||
|
|
}
|
||
|
|
|
||
|
|
try:
|
||
|
|
# 检查桶是否存在
|
||
|
|
if not self.rustfs.client.bucket_exists(bucket_name):
|
||
|
|
result['success'] = True
|
||
|
|
result['message'] = "桶不存在，无需删除"
|
||
|
|
logger.info(f"桶 {bucket_name} 不存在，跳过删除")
|
||
|
|
return result
|
||
|
|
|
||
|
|
# 列出所有文件
|
||
|
|
objects = list(self.rustfs.client.list_objects(bucket_name, recursive=True))
|
||
|
|
file_count = len(objects)
|
||
|
|
|
||
|
|
if file_count > 0:
|
||
|
|
logger.info(f"开始删除桶 {bucket_name} 中的 {file_count} 个文件")
|
||
|
|
|
||
|
|
# 删除所有文件
|
||
|
|
for obj in objects:
|
||
|
|
try:
|
||
|
|
self.rustfs.client.remove_object(bucket_name, obj.object_name)
|
||
|
|
result['files_deleted'] += 1
|
||
|
|
logger.debug(f"删除文件: {bucket_name}/{obj.object_name}")
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"删除文件失败 {bucket_name}/{obj.object_name}: {e}")
|
||
|
|
result['error'] = f"删除文件失败: {e}"
|
||
|
|
return result
|
||
|
|
|
||
|
|
# 删除空桶
|
||
|
|
self.rustfs.client.remove_bucket(bucket_name)
|
||
|
|
|
||
|
|
result['success'] = True
|
||
|
|
logger.info(f"桶 {bucket_name} 删除成功，共删除 {file_count} 个文件")
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
result['success'] = False
|
||
|
|
result['error'] = str(e)
|
||
|
|
logger.error(f"删除桶 {bucket_name} 失败: {e}")
|
||
|
|
|
||
|
|
return result
|
||
|
|
|
||
|
|
async def cleanup_all_old_buckets(self) -> Dict[str, Dict]:
|
||
|
|
"""清理所有旧桶"""
|
||
|
|
cleanup_results = {}
|
||
|
|
|
||
|
|
try:
|
||
|
|
# 检查所有桶状态
|
||
|
|
logger.info("=== 检查旧桶状态 ===")
|
||
|
|
bucket_status = await self.check_bucket_status()
|
||
|
|
|
||
|
|
# 统计需要清理的桶
|
||
|
|
buckets_to_clean = [
|
||
|
|
bucket_name for bucket_name, status in bucket_status.items()
|
||
|
|
if status['exists'] and status['file_count'] > 0
|
||
|
|
]
|
||
|
|
|
||
|
|
if not buckets_to_clean:
|
||
|
|
logger.info("没有需要清理的桶")
|
||
|
|
return {}
|
||
|
|
|
||
|
|
print(f"\n发现 {len(buckets_to_clean)} 个需要清理的桶:")
|
||
|
|
for bucket_name in buckets_to_clean:
|
||
|
|
status = bucket_status[bucket_name]
|
||
|
|
print(f" - {bucket_name}: {status['file_count']} 个文件, {status['total_size']} 字节")
|
||
|
|
|
||
|
|
# 确认清理
|
||
|
|
print("\n警告：此操作将永久删除这些桶及其所有文件！")
|
||
|
|
confirm = input("确认清理？(输入 'DELETE' 确认): ").strip()
|
||
|
|
|
||
|
|
if confirm != 'DELETE':
|
||
|
|
logger.info("用户取消清理操作")
|
||
|
|
return {}
|
||
|
|
|
||
|
|
# 执行清理
|
||
|
|
logger.info("=== 开始清理旧桶 ===")
|
||
|
|
|
||
|
|
for bucket_name in buckets_to_clean:
|
||
|
|
print(f"\n清理桶: {bucket_name}")
|
||
|
|
result = await self.delete_bucket(bucket_name)
|
||
|
|
cleanup_results[bucket_name] = result
|
||
|
|
|
||
|
|
if result['success']:
|
||
|
|
print(f" ✓ 清理成功，删除 {result['files_deleted']} 个文件")
|
||
|
|
else:
|
||
|
|
print(f" ✗ 清理失败: {result['error']}")
|
||
|
|
|
||
|
|
logger.info("=== 旧桶清理完成 ===")
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"清理过程失败: {e}")
|
||
|
|
|
||
|
|
return cleanup_results
|
||
|
|
|
||
|
|
async def run_cleanup(self) -> Dict[str, any]:
|
||
|
|
"""运行完整清理流程"""
|
||
|
|
cleanup_summary = {
|
||
|
|
'start_time': datetime.now().isoformat(),
|
||
|
|
'connection_status': False,
|
||
|
|
'all_buckets': [],
|
||
|
|
'bucket_status': {},
|
||
|
|
'cleanup_results': {},
|
||
|
|
'end_time': None,
|
||
|
|
'status': 'failed'
|
||
|
|
}
|
||
|
|
|
||
|
|
try:
|
||
|
|
# 连接
|
||
|
|
logger.info("=== RustFS 旧桶清理开始 ===")
|
||
|
|
connection_result = await self.connect()
|
||
|
|
if not connection_result:
|
||
|
|
raise RuntimeError("无法连接到 RustFS")
|
||
|
|
|
||
|
|
cleanup_summary['connection_status'] = True
|
||
|
|
|
||
|
|
# 列出所有桶
|
||
|
|
all_buckets = await self.list_all_buckets()
|
||
|
|
cleanup_summary['all_buckets'] = all_buckets
|
||
|
|
|
||
|
|
# 检查状态
|
||
|
|
bucket_status = await self.check_bucket_status()
|
||
|
|
cleanup_summary['bucket_status'] = bucket_status
|
||
|
|
|
||
|
|
# 执行清理
|
||
|
|
cleanup_results = await self.cleanup_all_old_buckets()
|
||
|
|
cleanup_summary['cleanup_results'] = cleanup_results
|
||
|
|
|
||
|
|
# 完成
|
||
|
|
cleanup_summary['status'] = 'completed'
|
||
|
|
cleanup_summary['end_time'] = datetime.now().isoformat()
|
||
|
|
|
||
|
|
logger.info("=== RustFS 旧桶清理完成 ===")
|
||
|
|
|
||
|
|
return cleanup_summary
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
cleanup_summary['error'] = str(e)
|
||
|
|
cleanup_summary['end_time'] = datetime.now().isoformat()
|
||
|
|
logger.error(f"清理失败: {e}")
|
||
|
|
return cleanup_summary
|
||
|
|
|
||
|
|
finally:
|
||
|
|
await self.close()
|
||
|
|
|
||
|
|
|
||
|
|
async def main():
|
||
|
|
"""主函数"""
|
||
|
|
cleanup = RustFSCleanup()
|
||
|
|
|
||
|
|
print("=== RustFS 旧桶清理工具 ===")
|
||
|
|
print("注意：这是rustFS，而不是minio，只是用了minio的通用S3接口")
|
||
|
|
print("\n此工具将删除以下遗留桶：")
|
||
|
|
print(" - moldinsight-geometry")
|
||
|
|
print(" - moldinsight-stp-files")
|
||
|
|
print(" - moldinsight-mold-cavities")
|
||
|
|
print(" - moldinsight-html")
|
||
|
|
print(" - moldinsight-user-files")
|
||
|
|
print("\n请确保所有重要数据已备份！")
|
||
|
|
|
||
|
|
# 运行清理
|
||
|
|
result = await cleanup.run_cleanup()
|
||
|
|
|
||
|
|
# 输出结果
|
||
|
|
print("\n=== 清理结果摘要 ===")
|
||
|
|
print(f"状态: {result['status']}")
|
||
|
|
print(f"开始时间: {result['start_time']}")
|
||
|
|
print(f"结束时间: {result['end_time']}")
|
||
|
|
|
||
|
|
if 'error' in result:
|
||
|
|
print(f"错误: {result['error']}")
|
||
|
|
|
||
|
|
# 桶状态
|
||
|
|
print("\n--- 桶状态检查 ---")
|
||
|
|
for bucket_name, status in result['bucket_status'].items():
|
||
|
|
if status['exists']:
|
||
|
|
print(f"{bucket_name}: 存在, {status['file_count']} 个文件")
|
||
|
|
else:
|
||
|
|
print(f"{bucket_name}: 不存在")
|
||
|
|
|
||
|
|
# 清理结果
|
||
|
|
print("\n--- 清理结果 ---")
|
||
|
|
if result['cleanup_results']:
|
||
|
|
for bucket_name, cleanup_result in result['cleanup_results'].items():
|
||
|
|
if cleanup_result['success']:
|
||
|
|
print(f"{bucket_name}: 成功，删除 {cleanup_result['files_deleted']} 个文件")
|
||
|
|
else:
|
||
|
|
print(f"{bucket_name}: 失败 - {cleanup_result.get('error', '未知错误')}")
|
||
|
|
else:
|
||
|
|
print("未执行清理操作")
|
||
|
|
|
||
|
|
print("\n=== 清理完成 ===")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
asyncio.run(main())
|