mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-10-12 02:16:39 +00:00
Feature: delete user all stuffs when delete user (#20010)
This commit is contained in:
+7
-190
@@ -74,6 +74,8 @@ type Service struct {
|
||||
BillingSubscriptionDAO *dao.BillingSubscriptionDAO
|
||||
MemoryDAO *dao.MemoryDAO
|
||||
SearchDAO *dao.SearchDAO
|
||||
deleteEngine engine.DocEngine
|
||||
deleteStorage storage.Storage
|
||||
}
|
||||
|
||||
// NewService create admin service
|
||||
@@ -524,12 +526,11 @@ type DeleteUserResult struct {
|
||||
// - *DeleteUserResult
|
||||
// - error: error message
|
||||
func (s *Service) DeleteUser(ctx context.Context, username string) (*DeleteUserResult, error) {
|
||||
result := &DeleteUserResult{
|
||||
Username: username,
|
||||
DeletedDetails: []string{fmt.Sprintf("Drop user: %s", username)},
|
||||
}
|
||||
userList, err := s.userDAO.ListByEmail(ctx, dao.DB, username)
|
||||
if err != nil || len(userList) == 0 {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get user: %w", err)
|
||||
}
|
||||
if len(userList) == 0 {
|
||||
return nil, fmt.Errorf("user '%s' not found", username)
|
||||
}
|
||||
|
||||
@@ -549,191 +550,7 @@ func (s *Service) DeleteUser(ctx context.Context, username string) (*DeleteUserR
|
||||
return nil, fmt.Errorf("user '%s' is admin account and cannot be deleted", username)
|
||||
}
|
||||
|
||||
// Get user-tenant relations
|
||||
tenants, err := s.userTenantDAO.GetByUserIDAll(ctx, dao.DB, user.ID)
|
||||
if err != nil {
|
||||
common.Warn("failed to get user-tenant relations", zap.Error(err))
|
||||
}
|
||||
|
||||
// Find owned tenant (role = "owner")
|
||||
var ownedTenantID string
|
||||
for _, t := range tenants {
|
||||
if t.Role == "owner" {
|
||||
ownedTenantID = t.TenantID
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Start transaction for cascade delete
|
||||
tx := dao.DB.Begin()
|
||||
if tx.Error != nil {
|
||||
return nil, fmt.Errorf("failed to begin transaction: %w", tx.Error)
|
||||
}
|
||||
|
||||
// Rollback helper function
|
||||
rollbackTx := func() {
|
||||
if rbErr := tx.Rollback(); rbErr.Error != nil {
|
||||
common.Error("failed to rollback transaction", rbErr.Error)
|
||||
}
|
||||
}
|
||||
|
||||
result.DeletedDetails = append(result.DeletedDetails, "Start to delete owned tenant.")
|
||||
// Delete owned tenant data
|
||||
if ownedTenantID != "" {
|
||||
// 1. Get knowledge base IDs
|
||||
kbIDs, err := s.kbDAO.GetKBIDsByTenantIDSimple(ctx, tx, ownedTenantID)
|
||||
if err != nil {
|
||||
common.Warn("failed to get knowledge base IDs", zap.Error(err))
|
||||
}
|
||||
|
||||
if len(kbIDs) > 0 {
|
||||
// 2. Get document IDs
|
||||
docIDs, err := s.documentDAO.GetAllDocIDsByKBIDs(ctx, tx, kbIDs)
|
||||
if err != nil {
|
||||
common.Warn("failed to get document IDs", zap.Error(err))
|
||||
}
|
||||
|
||||
// 3. Delete tasks by document IDs
|
||||
if len(docIDs) > 0 {
|
||||
docIDList := make([]string, len(docIDs))
|
||||
for i, d := range docIDs {
|
||||
docIDList[i] = d["id"]
|
||||
}
|
||||
if delErr := tx.Unscoped().Where("doc_id IN ?", docIDList).Delete(&entity.Task{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete tasks", zap.Error(delErr.Error))
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Delete documents
|
||||
if delErr := tx.Unscoped().Where("kb_id IN ?", kbIDs).Delete(&entity.Document{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete documents", zap.Error(delErr.Error))
|
||||
}
|
||||
|
||||
// 5. Delete knowledge bases
|
||||
if delErr := tx.Unscoped().Where("id IN ?", kbIDs).Delete(&entity.Knowledgebase{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete knowledge bases", zap.Error(delErr.Error))
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Delete files
|
||||
if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.File{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete files", zap.Error(delErr.Error))
|
||||
}
|
||||
|
||||
// 7. Delete user canvas (agents)
|
||||
if delErr := tx.Unscoped().Where("user_id = ?", ownedTenantID).Delete(&entity.UserCanvas{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete user canvas", zap.Error(delErr.Error))
|
||||
}
|
||||
|
||||
// 8. Get dialog IDs
|
||||
var dialogIDs []string
|
||||
if pluckErr := tx.Model(&entity.Chat{}).Where("tenant_id = ?", ownedTenantID).Pluck("id", &dialogIDs); pluckErr.Error != nil {
|
||||
common.Warn("failed to get dialog IDs", zap.Error(pluckErr.Error))
|
||||
}
|
||||
|
||||
// 9. Delete chat sessions
|
||||
if len(dialogIDs) > 0 {
|
||||
var sessionIDs []string
|
||||
if pluckErr := tx.Model(&entity.ChatSession{}).Where("dialog_id IN ?", dialogIDs).Pluck("id", &sessionIDs); pluckErr.Error != nil {
|
||||
common.Warn("failed to get chat session IDs", zap.Error(pluckErr.Error))
|
||||
}
|
||||
if len(sessionIDs) > 0 {
|
||||
if delErr := tx.Table("conversation_message").Where("conversation_id IN ?", sessionIDs).Delete(map[string]interface{}{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete conversation messages", zap.Error(delErr.Error))
|
||||
}
|
||||
if delErr := tx.Table("conversation_reference").Where("conversation_id IN ?", sessionIDs).Delete(map[string]interface{}{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete conversation references", zap.Error(delErr.Error))
|
||||
}
|
||||
}
|
||||
if delErr := tx.Unscoped().Where("dialog_id IN ?", dialogIDs).Delete(&entity.ChatSession{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete chat sessions", zap.Error(delErr.Error))
|
||||
}
|
||||
}
|
||||
|
||||
// 10. Delete chats/dialogs
|
||||
if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.Chat{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete chats", zap.Error(delErr.Error))
|
||||
}
|
||||
|
||||
// 11. Delete API tokens
|
||||
if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.APIToken{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete API tokens", zap.Error(delErr.Error))
|
||||
}
|
||||
|
||||
// 12. Delete API4Conversations
|
||||
if len(dialogIDs) > 0 {
|
||||
var sessionIDs []string
|
||||
if pluckErr := tx.Model(&entity.API4Conversation{}).Where("dialog_id IN ?", dialogIDs).Pluck("id", &sessionIDs); pluckErr.Error != nil {
|
||||
common.Warn("failed to get API conversation IDs", zap.Error(pluckErr.Error))
|
||||
}
|
||||
if len(sessionIDs) > 0 {
|
||||
if delErr := tx.Table("api_4_conversation_message").Where("conversation_id IN ?", sessionIDs).Delete(map[string]interface{}{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete API conversation messages", zap.Error(delErr.Error))
|
||||
}
|
||||
if delErr := tx.Table("api_4_conversation_reference").Where("conversation_id IN ?", sessionIDs).Delete(map[string]interface{}{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete API conversation references", zap.Error(delErr.Error))
|
||||
}
|
||||
}
|
||||
if delErr := tx.Unscoped().Where("dialog_id IN ?", dialogIDs).Delete(&entity.API4Conversation{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete API4Conversations", zap.Error(delErr.Error))
|
||||
}
|
||||
}
|
||||
|
||||
var tenantLLMCount int64
|
||||
tx.Model(&entity.TenantLLM{}).Where("tenant_id = ?", ownedTenantID).Count(&tenantLLMCount)
|
||||
result.TenantLLMCount = int(tenantLLMCount)
|
||||
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d tenant-LLM records.", tenantLLMCount))
|
||||
|
||||
result.LangfuseCount = 0
|
||||
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d langfuse records.", result.LangfuseCount))
|
||||
|
||||
metadataTableName := fmt.Sprintf("ragflow_doc_meta_%s", ownedTenantID[:32])
|
||||
result.MetadataTable = metadataTableName
|
||||
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted metadata table %s.", metadataTableName))
|
||||
|
||||
// 13. Delete tenant LLM configurations
|
||||
if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.TenantLLM{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete tenant LLM", zap.Error(delErr.Error))
|
||||
}
|
||||
|
||||
var tenantCount int64
|
||||
tx.Model(&entity.Tenant{}).Where("id = ?", ownedTenantID).Count(&tenantCount)
|
||||
result.TenantCount = int(tenantCount)
|
||||
// 14. Delete tenant
|
||||
if delErr := tx.Unscoped().Where("id = ?", ownedTenantID).Delete(&entity.Tenant{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete tenant", zap.Error(delErr.Error))
|
||||
}
|
||||
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d tenant.", result.TenantCount))
|
||||
}
|
||||
|
||||
var userTenantCount int64
|
||||
tx.Model(&entity.UserTenant{}).Where("user_id = ?", user.ID).Count(&userTenantCount)
|
||||
result.UserTenantCount = int(userTenantCount)
|
||||
|
||||
// 15. Delete user-tenant relations
|
||||
if delErr := tx.Unscoped().Where("user_id = ?", user.ID).Delete(&entity.UserTenant{}); delErr.Error != nil {
|
||||
common.Warn("failed to delete user-tenant relations", zap.Error(delErr.Error))
|
||||
}
|
||||
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d user-tenant records.", result.UserTenantCount))
|
||||
|
||||
result.UserCount = 1
|
||||
// 16. Finally, hard delete user
|
||||
if delErr := tx.Unscoped().Where("id = ?", user.ID).Delete(&entity.User{}); delErr.Error != nil {
|
||||
rollbackTx()
|
||||
return nil, fmt.Errorf("failed to delete user: %w", delErr.Error)
|
||||
}
|
||||
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d user.", result.UserCount))
|
||||
|
||||
// Commit transaction
|
||||
if commitErr := tx.Commit(); commitErr.Error != nil {
|
||||
return nil, fmt.Errorf("failed to commit transaction: %w", commitErr.Error)
|
||||
}
|
||||
|
||||
result.DeletedDetails = append(result.DeletedDetails, "Delete done!")
|
||||
|
||||
common.Info("Delete user success with all related data", zap.String("username", username))
|
||||
|
||||
return result, nil
|
||||
return s.deleteUserData(ctx, user)
|
||||
}
|
||||
|
||||
// ChangePassword change user password
|
||||
|
||||
@@ -0,0 +1,760 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package admin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/engine"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/ingestion/knowledge_compile"
|
||||
servicepkg "ragflow/internal/service"
|
||||
"ragflow/internal/storage"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type userDeletionData struct {
|
||||
ownedTenantID string
|
||||
datasets []entity.Knowledgebase
|
||||
documents []entity.Document
|
||||
files []entity.File
|
||||
memories []entity.Memory
|
||||
spaces []entity.SkillSpace
|
||||
datasetIDs []string
|
||||
documentIDs []string
|
||||
fileIDs []string
|
||||
docTenants map[string]string
|
||||
datasetNames map[string]string
|
||||
}
|
||||
|
||||
func namedID(name, id string) string {
|
||||
if name == "" {
|
||||
return id
|
||||
}
|
||||
return fmt.Sprintf("%s (%s)", name, id)
|
||||
}
|
||||
|
||||
func namedDocument(document entity.Document) string {
|
||||
if document.Name == nil {
|
||||
return document.ID
|
||||
}
|
||||
return namedID(*document.Name, document.ID)
|
||||
}
|
||||
|
||||
func (s *Service) deleteUserData(ctx context.Context, user *entity.User) (*DeleteUserResult, error) {
|
||||
data, err := loadUserDeletionData(ctx, user.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
docEngine := s.deleteEngine
|
||||
if docEngine == nil {
|
||||
docEngine = engine.Get()
|
||||
}
|
||||
store := s.deleteStorage
|
||||
if store == nil {
|
||||
store = storage.GetStorageFactory().GetStorage()
|
||||
}
|
||||
if err := data.deleteExternalData(ctx, docEngine, store); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := &DeleteUserResult{
|
||||
Username: user.Email,
|
||||
DeletedDetails: []string{fmt.Sprintf("Drop user: %s", user.Email)},
|
||||
}
|
||||
if err := dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
return data.deleteDatabaseRows(ctx, tx, user.ID, result)
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("failed to delete user data: %w", err)
|
||||
}
|
||||
datasetIDs := make(map[string]struct{}, len(data.datasetIDs))
|
||||
for _, id := range data.datasetIDs {
|
||||
datasetIDs[id] = struct{}{}
|
||||
}
|
||||
publishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
|
||||
defer cancel()
|
||||
publishWarning := false
|
||||
for _, document := range data.documents {
|
||||
if _, deletingDataset := datasetIDs[document.KbID]; deletingDataset {
|
||||
continue
|
||||
}
|
||||
if err := publishCtx.Err(); err != nil {
|
||||
common.Warn("stopped publishing document deletions", zap.Error(err))
|
||||
publishWarning = true
|
||||
break
|
||||
}
|
||||
if err := knowledge_compile.PublishDeleted(publishCtx, data.docTenants[document.KbID], document.KbID, document.ID, nil, nil); err != nil {
|
||||
common.Warn("failed to publish document deletion", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.Error(err))
|
||||
publishWarning = true
|
||||
}
|
||||
}
|
||||
if publishWarning {
|
||||
result.DeletedDetails = append(result.DeletedDetails, "- Warning: dataset artifact refresh could not be scheduled for some deleted documents.")
|
||||
}
|
||||
result.DeletedDetails = append(result.DeletedDetails, "Delete done!")
|
||||
common.Info("Delete user success with all related data")
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func loadUserDeletionData(ctx context.Context, userID string) (*userDeletionData, error) {
|
||||
db := dao.DB.WithContext(ctx)
|
||||
var tenants []entity.UserTenant
|
||||
if err := db.Where("user_id = ? AND role = ?", userID, "owner").Find(&tenants).Error; err != nil {
|
||||
return nil, fmt.Errorf("load owned tenant: %w", err)
|
||||
}
|
||||
if len(tenants) > 1 {
|
||||
return nil, fmt.Errorf("user %s owns more than one tenant", userID)
|
||||
}
|
||||
data := &userDeletionData{}
|
||||
if len(tenants) == 1 {
|
||||
data.ownedTenantID = tenants[0].TenantID
|
||||
}
|
||||
datasetQuery := db.Model(&entity.Knowledgebase{}).Select("id", "tenant_id", "created_by", "name").Where("created_by = ?", userID)
|
||||
if data.ownedTenantID != "" {
|
||||
datasetQuery = datasetQuery.Or("tenant_id = ?", data.ownedTenantID)
|
||||
}
|
||||
if err := datasetQuery.Find(&data.datasets).Error; err != nil {
|
||||
return nil, fmt.Errorf("load datasets: %w", err)
|
||||
}
|
||||
data.datasetNames = make(map[string]string, len(data.datasets))
|
||||
for _, dataset := range data.datasets {
|
||||
data.datasetIDs = append(data.datasetIDs, dataset.ID)
|
||||
data.datasetNames[dataset.ID] = dataset.Name
|
||||
}
|
||||
if err := db.Model(&entity.Document{}).Select("id", "kb_id", "name", "location", "token_num", "chunk_num").Where("created_by = ?", userID).Find(&data.documents).Error; err != nil {
|
||||
return nil, fmt.Errorf("load documents: %w", err)
|
||||
}
|
||||
seenDocuments := make(map[string]struct{}, len(data.documents))
|
||||
for _, document := range data.documents {
|
||||
seenDocuments[document.ID] = struct{}{}
|
||||
}
|
||||
for start := 0; start < len(data.datasetIDs); start += 1000 {
|
||||
end := min(start+1000, len(data.datasetIDs))
|
||||
var documents []entity.Document
|
||||
if err := db.Model(&entity.Document{}).Select("id", "kb_id", "name", "location", "token_num", "chunk_num").Where("kb_id IN ?", data.datasetIDs[start:end]).Find(&documents).Error; err != nil {
|
||||
return nil, fmt.Errorf("load dataset documents: %w", err)
|
||||
}
|
||||
for _, document := range documents {
|
||||
if _, seen := seenDocuments[document.ID]; !seen {
|
||||
data.documents = append(data.documents, document)
|
||||
seenDocuments[document.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, document := range data.documents {
|
||||
data.documentIDs = append(data.documentIDs, document.ID)
|
||||
}
|
||||
data.docTenants = make(map[string]string)
|
||||
if len(data.documents) > 0 {
|
||||
deletingDatasets := make(map[string]struct{}, len(data.datasetIDs))
|
||||
for _, id := range data.datasetIDs {
|
||||
deletingDatasets[id] = struct{}{}
|
||||
}
|
||||
remainingDatasetIDs := make(map[string]struct{})
|
||||
for _, document := range data.documents {
|
||||
if _, deleting := deletingDatasets[document.KbID]; !deleting {
|
||||
remainingDatasetIDs[document.KbID] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(remainingDatasetIDs) > 0 {
|
||||
ids := make([]string, 0, len(remainingDatasetIDs))
|
||||
for id := range remainingDatasetIDs {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
for start := 0; start < len(ids); start += 1000 {
|
||||
end := min(start+1000, len(ids))
|
||||
var datasets []entity.Knowledgebase
|
||||
if err := db.Select("id", "tenant_id", "name").Where("id IN ?", ids[start:end]).Find(&datasets).Error; err != nil {
|
||||
return nil, fmt.Errorf("load document tenants: %w", err)
|
||||
}
|
||||
for _, dataset := range datasets {
|
||||
data.docTenants[dataset.ID] = dataset.TenantID
|
||||
data.datasetNames[dataset.ID] = dataset.Name
|
||||
}
|
||||
}
|
||||
if len(data.docTenants) != len(ids) {
|
||||
return nil, fmt.Errorf("some documents refer to missing datasets")
|
||||
}
|
||||
}
|
||||
}
|
||||
fileQuery := db.Model(&entity.File{}).Select("id", "parent_id", "tenant_id", "name", "location", "type", "source_type").Where("created_by = ?", userID)
|
||||
if data.ownedTenantID != "" {
|
||||
fileQuery = fileQuery.Or("tenant_id = ?", data.ownedTenantID)
|
||||
}
|
||||
if err := fileQuery.Find(&data.files).Error; err != nil {
|
||||
return nil, fmt.Errorf("load files: %w", err)
|
||||
}
|
||||
for _, file := range data.files {
|
||||
data.fileIDs = append(data.fileIDs, file.ID)
|
||||
}
|
||||
if data.ownedTenantID != "" {
|
||||
if err := db.Select("id", "tenant_id").Where("tenant_id = ?", data.ownedTenantID).Find(&data.memories).Error; err != nil {
|
||||
return nil, fmt.Errorf("load memories: %w", err)
|
||||
}
|
||||
if err := db.Select("id", "tenant_id").Where("tenant_id = ?", data.ownedTenantID).Find(&data.spaces).Error; err != nil {
|
||||
return nil, fmt.Errorf("load skill spaces: %w", err)
|
||||
}
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (data *userDeletionData) deleteExternalData(ctx context.Context, docEngine engine.DocEngine, store storage.Storage) error {
|
||||
// External stores cannot join the SQL transaction. Keep the user row until
|
||||
// cleanup succeeds so a failed request can be retried.
|
||||
if docEngine == nil && (data.ownedTenantID != "" || len(data.datasets) > 0 || len(data.documents) > 0) {
|
||||
return fmt.Errorf("document engine is unavailable for user data cleanup")
|
||||
}
|
||||
if store == nil && (len(data.datasets) > 0 || len(data.documents) > 0 || len(data.files) > 0) {
|
||||
return fmt.Errorf("storage is unavailable for user data cleanup")
|
||||
}
|
||||
datasetIDs := make(map[string]struct{}, len(data.datasetIDs))
|
||||
for _, id := range data.datasetIDs {
|
||||
datasetIDs[id] = struct{}{}
|
||||
}
|
||||
for _, document := range data.documents {
|
||||
if document.Location == nil || *document.Location == "" {
|
||||
continue
|
||||
}
|
||||
exists, err := store.ObjectExists(ctx, document.KbID, *document.Location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check document %s: %w", document.ID, err)
|
||||
}
|
||||
if !exists {
|
||||
common.Warn("Document object already missing", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))
|
||||
continue
|
||||
}
|
||||
if err := store.Remove(ctx, document.KbID, *document.Location); err != nil {
|
||||
return fmt.Errorf("remove document %s: %w", document.ID, err)
|
||||
}
|
||||
common.Info("Removed document object", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))
|
||||
}
|
||||
for _, dataset := range data.datasets {
|
||||
exists, err := store.BucketExistsWithError(ctx, dataset.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check dataset bucket %s: %w", dataset.ID, err)
|
||||
}
|
||||
if !exists {
|
||||
common.Warn("Dataset bucket already missing", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID))
|
||||
continue
|
||||
}
|
||||
if err := store.RemoveEmptyBucket(ctx, dataset.ID); err != nil {
|
||||
common.Warn("Unable to remove empty dataset bucket", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID), zap.Error(err))
|
||||
} else {
|
||||
common.Info("Removed empty dataset bucket", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID))
|
||||
}
|
||||
}
|
||||
for _, file := range data.files {
|
||||
if file.SourceType != string(entity.FileSourceKnowledgebase) && file.Location != nil && *file.Location != "" && file.Type != "folder" {
|
||||
exists, err := store.ObjectExists(ctx, file.ParentID, *file.Location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check file %s: %w", file.ID, err)
|
||||
}
|
||||
if !exists {
|
||||
common.Warn("File object already missing", zap.String("file", namedID(file.Name, file.ID)), zap.String("file_id", file.ID), zap.String("bucket", file.ParentID))
|
||||
continue
|
||||
}
|
||||
if err := store.Remove(ctx, file.ParentID, *file.Location); err != nil {
|
||||
return fmt.Errorf("remove file %s: %w", file.ID, err)
|
||||
}
|
||||
common.Info("Removed file object", zap.String("file", namedID(file.Name, file.ID)), zap.String("file_id", file.ID), zap.String("bucket", file.ParentID))
|
||||
}
|
||||
}
|
||||
for _, file := range data.files {
|
||||
if file.Type != "folder" || file.SourceType == string(entity.FileSourceKnowledgebase) || file.TenantID != data.ownedTenantID {
|
||||
continue
|
||||
}
|
||||
exists, err := store.BucketExistsWithError(ctx, file.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check folder bucket %s: %w", file.ID, err)
|
||||
}
|
||||
if !exists {
|
||||
common.Warn("Folder bucket already missing", zap.String("folder", namedID(file.Name, file.ID)), zap.String("bucket", file.ID))
|
||||
continue
|
||||
}
|
||||
if err := store.RemoveEmptyBucket(ctx, file.ID); err != nil {
|
||||
common.Warn("Failed to remove empty folder bucket", zap.String("folder", namedID(file.Name, file.ID)), zap.String("bucket", file.ID), zap.Error(err))
|
||||
} else {
|
||||
common.Info("Removed empty folder bucket", zap.String("folder", namedID(file.Name, file.ID)), zap.String("bucket", file.ID))
|
||||
}
|
||||
}
|
||||
if docEngine == nil {
|
||||
return nil
|
||||
}
|
||||
if data.ownedTenantID != "" {
|
||||
ownedDatasetIDs := make([]string, 0)
|
||||
for _, dataset := range data.datasets {
|
||||
if dataset.TenantID == data.ownedTenantID {
|
||||
ownedDatasetIDs = append(ownedDatasetIDs, dataset.ID)
|
||||
}
|
||||
}
|
||||
if err := dropUserChunkStore(ctx, docEngine, servicepkg.IndexName(data.ownedTenantID), ownedDatasetIDs); err != nil {
|
||||
return fmt.Errorf("remove tenant chunks: %w", err)
|
||||
}
|
||||
if err := dropUserChunkStore(ctx, docEngine, servicepkg.MemoryIndexName(data.ownedTenantID), memoryIDs(data.memories)); err != nil {
|
||||
return fmt.Errorf("remove memory messages: %w", err)
|
||||
}
|
||||
if err := docEngine.DropMetadataStore(ctx, data.ownedTenantID); err != nil {
|
||||
return fmt.Errorf("remove tenant metadata: %w", err)
|
||||
}
|
||||
}
|
||||
for _, dataset := range data.datasets {
|
||||
if dataset.TenantID != data.ownedTenantID {
|
||||
indexName := servicepkg.IndexName(dataset.TenantID)
|
||||
exists, err := docEngine.ChunkStoreExists(ctx, indexName, dataset.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check dataset chunks %s: %w", dataset.ID, err)
|
||||
}
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
if _, err := docEngine.DeleteChunks(ctx, map[string]interface{}{"kb_id": dataset.ID}, indexName, dataset.ID); err != nil {
|
||||
return fmt.Errorf("remove dataset chunks %s: %w", dataset.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
documentsByDataset := make(map[string][]string)
|
||||
for _, document := range data.documents {
|
||||
if _, deletingDataset := datasetIDs[document.KbID]; !deletingDataset {
|
||||
documentsByDataset[document.KbID] = append(documentsByDataset[document.KbID], document.ID)
|
||||
}
|
||||
}
|
||||
for kbID, docIDs := range documentsByDataset {
|
||||
tenantID := data.docTenants[kbID]
|
||||
if tenantID == "" {
|
||||
return fmt.Errorf("dataset %s for documents not found", kbID)
|
||||
}
|
||||
indexName := servicepkg.IndexName(tenantID)
|
||||
exists, err := docEngine.ChunkStoreExists(ctx, indexName, kbID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check document chunks for dataset %s: %w", kbID, err)
|
||||
}
|
||||
if exists {
|
||||
for start := 0; start < len(docIDs); start += 1000 {
|
||||
end := min(start+1000, len(docIDs))
|
||||
if _, err := docEngine.DeleteChunks(ctx, map[string]interface{}{"doc_id": docIDs[start:end]}, indexName, kbID); err != nil {
|
||||
return fmt.Errorf("remove document chunks for dataset %s: %w", kbID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
metadataExists, err := docEngine.MetadataStoreExists(ctx, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check document metadata for dataset %s: %w", kbID, err)
|
||||
}
|
||||
if metadataExists {
|
||||
for start := 0; start < len(docIDs); start += 1000 {
|
||||
end := min(start+1000, len(docIDs))
|
||||
if _, err := docEngine.DeleteMetadata(ctx, map[string]interface{}{"id": docIDs[start:end]}, tenantID); err != nil {
|
||||
return fmt.Errorf("remove document metadata for dataset %s: %w", kbID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, space := range data.spaces {
|
||||
if err := docEngine.DropChunkStore(ctx, servicepkg.SkillIndexName(space.TenantID, space.ID), "skill"); err != nil {
|
||||
return fmt.Errorf("remove skill index %s: %w", space.ID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dropUserChunkStore(ctx context.Context, docEngine engine.DocEngine, indexName string, ids []string) error {
|
||||
if docEngine.GetType() == string(engine.EngineInfinity) {
|
||||
for _, id := range ids {
|
||||
if err := docEngine.DropChunkStore(ctx, indexName, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return docEngine.DropChunkStore(ctx, indexName, "")
|
||||
}
|
||||
|
||||
func memoryIDs(memories []entity.Memory) []string {
|
||||
ids := make([]string, 0, len(memories))
|
||||
for _, memory := range memories {
|
||||
ids = append(ids, memory.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func (data *userDeletionData) deleteDatabaseRows(ctx context.Context, tx *gorm.DB, userID string, result *DeleteUserResult) error {
|
||||
selectIDs := func(model interface{}, condition string, args ...interface{}) ([]string, error) {
|
||||
var ids []string
|
||||
err := tx.WithContext(ctx).Model(model).Where(condition, args...).Pluck("id", &ids).Error
|
||||
return ids, err
|
||||
}
|
||||
remove := func(label string, model interface{}, condition string, args ...interface{}) (int64, error) {
|
||||
deleted := tx.WithContext(ctx).Unscoped().Where(condition, args...).Delete(model)
|
||||
if deleted.Error != nil {
|
||||
return 0, fmt.Errorf("delete %s: %w", label, deleted.Error)
|
||||
}
|
||||
if deleted.RowsAffected > 0 {
|
||||
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d %s records.", deleted.RowsAffected, label))
|
||||
}
|
||||
return deleted.RowsAffected, nil
|
||||
}
|
||||
removeIDs := func(label string, model interface{}, column string, ids []string) error {
|
||||
var total int64
|
||||
for start := 0; start < len(ids); start += 1000 {
|
||||
end := min(start+1000, len(ids))
|
||||
deleted := tx.WithContext(ctx).Unscoped().Where(column+" IN ?", ids[start:end]).Delete(model)
|
||||
if deleted.Error != nil {
|
||||
return fmt.Errorf("delete %s: %w", label, deleted.Error)
|
||||
}
|
||||
total += deleted.RowsAffected
|
||||
}
|
||||
if total > 0 {
|
||||
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d %s records.", total, label))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
owner := data.ownedTenantID
|
||||
var chatIDs, canvasIDs, conversationIDs, apiConversationIDs []string
|
||||
var evaluationDatasetIDs, evaluationRunIDs, connectorIDs, providerIDs, instanceIDs, modelIDs, groupIDs []string
|
||||
var ingestionTaskIDs, memoryTaskIDs, commitIDs []string
|
||||
var err error
|
||||
if owner != "" {
|
||||
if chatIDs, err = selectIDs(&entity.Chat{}, "tenant_id = ?", owner); err != nil {
|
||||
return err
|
||||
}
|
||||
if evaluationDatasetIDs, err = selectIDs(&entity.EvaluationDataset{}, "tenant_id = ? OR created_by = ?", owner, userID); err != nil {
|
||||
return err
|
||||
}
|
||||
if connectorIDs, err = selectIDs(&entity.Connector{}, "tenant_id = ?", owner); err != nil {
|
||||
return err
|
||||
}
|
||||
if providerIDs, err = selectIDs(&entity.TenantModelProvider{}, "tenant_id = ?", owner); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if evaluationDatasetIDs, err = selectIDs(&entity.EvaluationDataset{}, "created_by = ?", userID); err != nil {
|
||||
return err
|
||||
}
|
||||
if canvasIDs, err = selectIDs(&entity.UserCanvas{}, "user_id = ?", userID); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(chatIDs) > 0 {
|
||||
conversationIDs, err = selectIDs(&entity.ChatSession{}, "dialog_id IN ? OR user_id = ?", chatIDs, userID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
conversationIDs, err = selectIDs(&entity.ChatSession{}, "user_id = ?", userID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
apiDialogIDs := append(append([]string{}, chatIDs...), canvasIDs...)
|
||||
if len(apiDialogIDs) > 0 {
|
||||
apiConversationIDs, err = selectIDs(&entity.API4Conversation{}, "dialog_id IN ? OR user_id = ?", apiDialogIDs, userID)
|
||||
} else {
|
||||
apiConversationIDs, err = selectIDs(&entity.API4Conversation{}, "user_id = ?", userID)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(evaluationDatasetIDs) > 0 {
|
||||
evaluationRunIDs, err = selectIDs(&entity.EvaluationRun{}, "dataset_id IN ? OR created_by = ?", evaluationDatasetIDs, userID)
|
||||
} else {
|
||||
evaluationRunIDs, err = selectIDs(&entity.EvaluationRun{}, "created_by = ?", userID)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(providerIDs) > 0 {
|
||||
if instanceIDs, err = selectIDs(&entity.TenantModelInstance{}, "provider_id IN ?", providerIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if modelIDs, err = selectIDs(&entity.TenantModel{}, "provider_id IN ?", providerIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = tx.Model(&entity.TenantModelGroupMapping{}).Where("provider_id IN ?", providerIDs).Distinct().Pluck("group_id", &groupIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(data.documentIDs) > 0 || len(data.datasetIDs) > 0 {
|
||||
query := tx.Model(&entity.IngestionTask{}).Where("user_id = ?", userID)
|
||||
if len(data.documentIDs) > 0 {
|
||||
query = query.Or("document_id IN ?", data.documentIDs)
|
||||
}
|
||||
if len(data.datasetIDs) > 0 {
|
||||
query = query.Or("dataset_id IN ?", data.datasetIDs)
|
||||
}
|
||||
err = query.Pluck("id", &ingestionTaskIDs).Error
|
||||
} else {
|
||||
ingestionTaskIDs, err = selectIDs(&entity.IngestionTask{}, "user_id = ?", userID)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(data.memories) > 0 {
|
||||
if err = tx.Model(&entity.MemoryTask{}).Where("memory_id IN ?", memoryIDs(data.memories)).Pluck("task_id", &memoryTaskIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
commitQuery := tx.Model(&entity.FileCommit{}).Where("author_id = ?", userID)
|
||||
if len(data.fileIDs) > 0 {
|
||||
commitQuery = commitQuery.Or("folder_id IN ?", data.fileIDs)
|
||||
}
|
||||
if err = commitQuery.Pluck("id", &commitIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, child := range []struct {
|
||||
label string
|
||||
model interface{}
|
||||
ids []string
|
||||
}{
|
||||
{"conversation messages", &entity.ConversationMessage{}, conversationIDs},
|
||||
{"conversation references", &entity.ConversationReference{}, conversationIDs},
|
||||
{"API conversation messages", &entity.API4ConversationMessage{}, apiConversationIDs},
|
||||
{"API conversation references", &entity.API4ConversationReference{}, apiConversationIDs},
|
||||
} {
|
||||
if err := removeIDs(child.label, child.model, "conversation_id", child.ids); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := removeIDs("conversations", &entity.ChatSession{}, "id", conversationIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("API conversations", &entity.API4Conversation{}, "id", apiConversationIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("chat channels", &entity.ChatChannel{}, "chat_id", chatIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("chats", &entity.Chat{}, "id", chatIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("agent versions", &entity.UserCanvasVersion{}, "user_canvas_id", canvasIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("agents", &entity.UserCanvas{}, "id", canvasIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("evaluation results", &entity.EvaluationResult{}, "run_id", evaluationRunIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("evaluation runs", &entity.EvaluationRun{}, "id", evaluationRunIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(evaluationDatasetIDs) > 0 {
|
||||
var caseIDs []string
|
||||
if caseIDs, err = selectIDs(&entity.EvaluationCase{}, "dataset_id IN ?", evaluationDatasetIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("evaluation results", &entity.EvaluationResult{}, "case_id", caseIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("evaluation cases", &entity.EvaluationCase{}, "id", caseIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("evaluation datasets", &entity.EvaluationDataset{}, "id", evaluationDatasetIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := removeIDs("ingestion task logs", &entity.IngestionTaskLog{}, "task_id", ingestionTaskIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("ingestion tasks", &entity.IngestionTask{}, "id", ingestionTaskIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("memory tasks", &entity.MemoryTask{}, "memory_id", memoryIDs(data.memories)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("memory task records", &entity.Task{}, "id", memoryTaskIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("document tasks", &entity.Task{}, "doc_id", data.documentIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("document file links", &entity.File2Document{}, "document_id", data.documentIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("file document links", &entity.File2Document{}, "file_id", data.fileIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("documents", &entity.Document{}, "id", data.documentIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("file commit items", &entity.FileCommitItem{}, "commit_id", commitIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("file commit items", &entity.FileCommitItem{}, "file_id", data.fileIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("file commits", &entity.FileCommit{}, "id", commitIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("files", &entity.File{}, "id", data.fileIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("connector dataset links", &entity.Connector2Kb{}, "connector_id", connectorIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("connector dataset links", &entity.Connector2Kb{}, "kb_id", data.datasetIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("sync logs", &entity.SyncLogs{}, "connector_id", connectorIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("sync logs", &entity.SyncLogs{}, "kb_id", data.datasetIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("connectors", &entity.Connector{}, "id", connectorIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("knowledge compile records", &entity.KnowledgeCompileDataset{}, "dataset_id", data.datasetIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("wiki document records", &entity.WikiDocumentDirty{}, "document_id", data.documentIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("wiki document records", &entity.WikiDocumentDirty{}, "dataset_id", data.datasetIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("pipeline logs", &entity.PipelineOperationLog{}, "document_id", data.documentIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("pipeline logs", &entity.PipelineOperationLog{}, "kb_id", data.datasetIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("datasets", &entity.Knowledgebase{}, "id", data.datasetIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
datasetIDs := make(map[string]struct{}, len(data.datasetIDs))
|
||||
for _, id := range data.datasetIDs {
|
||||
datasetIDs[id] = struct{}{}
|
||||
}
|
||||
counts := make(map[string]struct{ documents, chunks, tokens int64 })
|
||||
for _, document := range data.documents {
|
||||
if _, deletingDataset := datasetIDs[document.KbID]; deletingDataset {
|
||||
continue
|
||||
}
|
||||
count := counts[document.KbID]
|
||||
count.documents++
|
||||
count.chunks += document.ChunkNum
|
||||
count.tokens += document.TokenNum
|
||||
counts[document.KbID] = count
|
||||
}
|
||||
for kbID, count := range counts {
|
||||
if err := tx.Model(&entity.Knowledgebase{}).Where("id = ?", kbID).Updates(map[string]interface{}{
|
||||
"doc_num": gorm.Expr("CASE WHEN doc_num >= ? THEN doc_num - ? ELSE 0 END", count.documents, count.documents),
|
||||
"chunk_num": gorm.Expr("CASE WHEN chunk_num >= ? THEN chunk_num - ? ELSE 0 END", count.chunks, count.chunks),
|
||||
"token_num": gorm.Expr("CASE WHEN token_num >= ? THEN token_num - ? ELSE 0 END", count.tokens, count.tokens),
|
||||
}).Error; err != nil {
|
||||
return fmt.Errorf("update dataset counters: %w", err)
|
||||
}
|
||||
}
|
||||
if len(groupIDs) > 0 {
|
||||
if err := removeIDs("model group mappings", &entity.TenantModelGroupMapping{}, "provider_id", providerIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, groupID := range groupIDs {
|
||||
var remaining int64
|
||||
if err := tx.Model(&entity.TenantModelGroupMapping{}).Where("group_id = ?", groupID).Count(&remaining).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if remaining == 0 {
|
||||
if _, err := remove("model groups", &entity.TenantModelGroup{}, "id = ?", groupID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := removeIDs("models", &entity.TenantModel{}, "id", modelIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("model instances", &entity.TenantModelInstance{}, "id", instanceIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := removeIDs("model providers", &entity.TenantModelProvider{}, "id", providerIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if owner != "" {
|
||||
for _, item := range []struct {
|
||||
label string
|
||||
model interface{}
|
||||
}{
|
||||
{"chat channels", &entity.ChatChannel{}},
|
||||
{"searches", &entity.Search{}},
|
||||
{"memories", &entity.Memory{}},
|
||||
{"MCP servers", &entity.MCPServer{}},
|
||||
{"skill search configurations", &entity.SkillSearchConfig{}},
|
||||
{"skill spaces", &entity.SkillSpace{}},
|
||||
{"compilation templates", &entity.CompilationTemplate{}},
|
||||
{"compilation template groups", &entity.CompilationTemplateGroup{}},
|
||||
{"pipeline logs", &entity.PipelineOperationLog{}},
|
||||
{"knowledge compile records", &entity.KnowledgeCompileDataset{}},
|
||||
{"wiki document records", &entity.WikiDocumentDirty{}},
|
||||
{"API tokens", &entity.APIToken{}},
|
||||
} {
|
||||
if _, err := remove(item.label, item.model, "tenant_id = ?", owner); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
tenantLLMCount, err := remove("tenant LLMs", &entity.TenantLLM{}, "tenant_id = ?", owner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.TenantLLMCount = int(tenantLLMCount)
|
||||
langfuseCount, err := remove("Langfuse credentials", &entity.TenantLangfuse{}, "tenant_id = ?", owner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.LangfuseCount = int(langfuseCount)
|
||||
result.MetadataTable = servicepkg.BuildMetadataIndexName(owner)
|
||||
ownerMemberships, err := remove("tenant memberships", &entity.UserTenant{}, "tenant_id = ?", owner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.UserTenantCount += int(ownerMemberships)
|
||||
count, err := remove("tenants", &entity.Tenant{}, "id = ?", owner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.TenantCount = int(count)
|
||||
}
|
||||
if _, err := remove("searches", &entity.Search{}, "created_by = ?", userID); err != nil {
|
||||
return err
|
||||
}
|
||||
invitationCondition := "user_id = ?"
|
||||
invitationArgs := []interface{}{userID}
|
||||
if owner != "" {
|
||||
invitationCondition += " OR tenant_id = ?"
|
||||
invitationArgs = append(invitationArgs, owner)
|
||||
}
|
||||
if _, err := remove("invitations", &entity.InvitationCode{}, invitationCondition, invitationArgs...); err != nil {
|
||||
return err
|
||||
}
|
||||
count, err := remove("user tenant memberships", &entity.UserTenant{}, "user_id = ?", userID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.UserTenantCount += int(count)
|
||||
count, err = remove("users", &entity.User{}, "id = ?", userID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.UserCount = int(count)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package admin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/engine"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/ingestion/testutil"
|
||||
"ragflow/internal/storage"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
"go.uber.org/zap/zaptest/observer"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type deletionEngine struct {
|
||||
engine.DocEngine
|
||||
dropped []string
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (e *deletionEngine) GetType() string { return string(engine.EngineElasticsearch) }
|
||||
func (e *deletionEngine) DropChunkStore(_ context.Context, name, _ string) error {
|
||||
e.dropped = append(e.dropped, name)
|
||||
return nil
|
||||
}
|
||||
func (e *deletionEngine) DropMetadataStore(_ context.Context, tenantID string) error {
|
||||
e.dropped = append(e.dropped, "metadata:"+tenantID)
|
||||
return nil
|
||||
}
|
||||
func (e *deletionEngine) ChunkStoreExists(context.Context, string, string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
func (e *deletionEngine) MetadataStoreExists(context.Context, string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
func (e *deletionEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, _, _ string) (int64, error) {
|
||||
for key, value := range condition {
|
||||
e.deleted = append(e.deleted, fmt.Sprintf("%s:%v", key, value))
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
func (e *deletionEngine) DeleteMetadata(_ context.Context, condition map[string]interface{}, _ string) (int64, error) {
|
||||
for key, value := range condition {
|
||||
e.deleted = append(e.deleted, fmt.Sprintf("metadata:%s:%v", key, value))
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
type deletionStorage struct {
|
||||
storage.Storage
|
||||
buckets []string
|
||||
files []string
|
||||
err error
|
||||
bucketErr error
|
||||
missing bool
|
||||
checkErr error
|
||||
}
|
||||
|
||||
func (s *deletionStorage) RemoveBucket(_ context.Context, bucket string) error {
|
||||
if s.err != nil {
|
||||
return s.err
|
||||
}
|
||||
s.buckets = append(s.buckets, bucket)
|
||||
return nil
|
||||
}
|
||||
func (s *deletionStorage) RemoveEmptyBucket(_ context.Context, bucket string) error {
|
||||
if s.bucketErr != nil {
|
||||
return s.bucketErr
|
||||
}
|
||||
s.buckets = append(s.buckets, bucket)
|
||||
return nil
|
||||
}
|
||||
func (s *deletionStorage) Remove(_ context.Context, bucket, name string, _ ...string) error {
|
||||
if s.err != nil {
|
||||
return s.err
|
||||
}
|
||||
s.files = append(s.files, bucket+"/"+name)
|
||||
return nil
|
||||
}
|
||||
func (s *deletionStorage) ObjectExists(_ context.Context, bucket, name string) (bool, error) {
|
||||
if s.checkErr != nil {
|
||||
return false, s.checkErr
|
||||
}
|
||||
return !s.missing, nil
|
||||
}
|
||||
func (s *deletionStorage) BucketExistsWithError(_ context.Context, bucket string) (bool, error) {
|
||||
if s.checkErr != nil {
|
||||
return false, s.checkErr
|
||||
}
|
||||
return !s.missing, nil
|
||||
}
|
||||
|
||||
func setupUserDeletionDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
db := testutil.SetupTestDB(t,
|
||||
&entity.User{}, &entity.Tenant{}, &entity.UserTenant{},
|
||||
&entity.Knowledgebase{}, &entity.Document{}, &entity.Task{}, &entity.File{}, &entity.File2Document{},
|
||||
&entity.Chat{}, &entity.ChatSession{}, &entity.ConversationMessage{}, &entity.ConversationReference{},
|
||||
&entity.API4Conversation{}, &entity.API4ConversationMessage{}, &entity.API4ConversationReference{},
|
||||
&entity.ChatChannel{}, &entity.UserCanvas{}, &entity.UserCanvasVersion{},
|
||||
&entity.Search{}, &entity.Memory{}, &entity.MemoryTask{}, &entity.MCPServer{},
|
||||
&entity.SkillSpace{}, &entity.SkillSearchConfig{},
|
||||
&entity.CompilationTemplate{}, &entity.CompilationTemplateGroup{},
|
||||
&entity.EvaluationDataset{}, &entity.EvaluationCase{}, &entity.EvaluationRun{}, &entity.EvaluationResult{},
|
||||
&entity.Connector{}, &entity.Connector2Kb{}, &entity.SyncLogs{},
|
||||
&entity.KnowledgeCompileDataset{}, &entity.WikiDocumentDirty{}, &entity.PipelineOperationLog{},
|
||||
&entity.IngestionTask{}, &entity.IngestionTaskLog{}, &entity.FileCommit{}, &entity.FileCommitItem{},
|
||||
&entity.TenantModelProvider{}, &entity.TenantModelInstance{}, &entity.TenantModel{},
|
||||
&entity.TenantModelGroup{}, &entity.TenantModelGroupMapping{},
|
||||
&entity.TenantLLM{}, &entity.TenantLangfuse{}, &entity.APIToken{}, &entity.InvitationCode{},
|
||||
)
|
||||
t.Cleanup(testutil.ReplaceDBForTest(t, db))
|
||||
return db
|
||||
}
|
||||
|
||||
func TestDeleteUserRemovesOwnedDataAndJoinedDocuments(t *testing.T) {
|
||||
db := setupUserDeletionDB(t)
|
||||
insert := func(query string, args ...interface{}) {
|
||||
t.Helper()
|
||||
if err := db.Exec(query, args...).Error; err != nil {
|
||||
t.Fatalf("insert fixture: %v", err)
|
||||
}
|
||||
}
|
||||
insert("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0")
|
||||
insert("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-2", "[email protected]", "Other", "1", "1", "0")
|
||||
insert("INSERT INTO tenant (id, llm_id, embd_id, asr_id, img2txt_id, rerank_id, parser_ids) VALUES (?, ?, ?, ?, ?, ?, ?)", "user-1", "llm", "embd", "asr", "img", "rerank", "general")
|
||||
insert("INSERT INTO user_tenant (id, user_id, tenant_id, role, invited_by) VALUES (?, ?, ?, ?, ?)", "owner", "user-1", "user-1", "owner", "user-1")
|
||||
insert("INSERT INTO user_tenant (id, user_id, tenant_id, role, invited_by) VALUES (?, ?, ?, ?, ?)", "joined", "user-1", "team", "normal", "user-2")
|
||||
insert("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id, doc_num, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "own-kb", "user-1", "Own", "embd", "me", "user-1", "general", 1, 2, 3)
|
||||
insert("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id, doc_num, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "team-kb", "team", "Team", "embd", "team", "user-2", "general", 2, 5, 7)
|
||||
insert("INSERT INTO document (id, kb_id, name, parser_id, parser_config, type, created_by, suffix, location, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "own-doc", "own-kb", "Owned document", "general", "{}", "pdf", "user-1", "pdf", "own.pdf", 2, 3)
|
||||
insert("INSERT INTO document (id, kb_id, name, parser_id, parser_config, type, created_by, suffix, location, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "joined-doc", "team-kb", "Joined document", "general", "{}", "pdf", "user-1", "pdf", "joined.pdf", 2, 3)
|
||||
insert("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", "other-doc", "team-kb", "general", "{}", "pdf", "user-2", "pdf", 3, 4)
|
||||
insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, location, type) VALUES (?, ?, ?, ?, ?, ?, ?)", "own-file", "own-folder", "user-1", "user-1", "own.pdf", "own.pdf", "file")
|
||||
insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, type) VALUES (?, ?, ?, ?, ?, ?)", "own-folder", "own-folder", "user-1", "user-1", "/", "folder")
|
||||
insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, location, type, source_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "kb-file", "kb-folder", "user-1", "user-1", "own.pdf", "own.pdf", "pdf", "knowledgebase")
|
||||
insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, location, type) VALUES (?, ?, ?, ?, ?, ?, ?)", "joined-file", "team-folder", "team", "user-1", "joined.pdf", "joined.pdf", "file")
|
||||
insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, location, type) VALUES (?, ?, ?, ?, ?, ?, ?)", "other-file", "team-folder", "team", "user-2", "other.pdf", "other.pdf", "file")
|
||||
insert("INSERT INTO file2document (id, file_id, document_id) VALUES (?, ?, ?)", "own-link", "own-file", "own-doc")
|
||||
insert("INSERT INTO file2document (id, file_id, document_id) VALUES (?, ?, ?)", "joined-link", "joined-file", "joined-doc")
|
||||
insert("INSERT INTO file2document (id, file_id, document_id) VALUES (?, ?, ?)", "shared-link", "other-file", "joined-doc")
|
||||
insert("INSERT INTO file2document (id, file_id, document_id) VALUES (?, ?, ?)", "other-link", "other-file", "other-doc")
|
||||
insert("INSERT INTO search (id, tenant_id, name, created_by, search_config) VALUES (?, ?, ?, ?, ?)", "own-search", "user-1", "Own", "user-1", "{}")
|
||||
insert("INSERT INTO search (id, tenant_id, name, created_by, search_config) VALUES (?, ?, ?, ?, ?)", "joined-search", "team", "Joined", "user-1", "{}")
|
||||
insert("INSERT INTO search (id, tenant_id, name, created_by, search_config) VALUES (?, ?, ?, ?, ?)", "other-search", "team", "Other", "user-2", "{}")
|
||||
insert("INSERT INTO dialog (id, tenant_id, llm_id, llm_setting, prompt_config, do_refer, kb_ids) VALUES (?, ?, ?, ?, ?, ?, ?)", "own-chat", "user-1", "llm", "{}", "{}", "1", "[]")
|
||||
insert("INSERT INTO dialog (id, tenant_id, llm_id, llm_setting, prompt_config, do_refer, kb_ids) VALUES (?, ?, ?, ?, ?, ?, ?)", "team-chat", "team", "llm", "{}", "{}", "1", "[]")
|
||||
insert("INSERT INTO conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "own-conversation", "own-chat", "user-1")
|
||||
insert("INSERT INTO conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "joined-conversation", "team-chat", "user-1")
|
||||
insert("INSERT INTO conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "other-conversation", "team-chat", "user-2")
|
||||
insert("INSERT INTO conversation_message (conversation_id, position, metadata) VALUES (?, ?, ?)", "own-conversation", 0, "{}")
|
||||
insert("INSERT INTO conversation_reference (conversation_id, position, reference) VALUES (?, ?, ?)", "own-conversation", 0, "{}")
|
||||
insert("INSERT INTO api_4_conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "own-api-conversation", "own-chat", "user-1")
|
||||
insert("INSERT INTO api_4_conversation_message (conversation_id, position, metadata) VALUES (?, ?, ?)", "own-api-conversation", 0, "{}")
|
||||
insert("INSERT INTO api_4_conversation_reference (conversation_id, position, reference) VALUES (?, ?, ?)", "own-api-conversation", 0, "{}")
|
||||
insert("INSERT INTO user_canvas (id, user_id, tags, canvas_category) VALUES (?, ?, ?, ?)", "agent", "user-1", "", "agent_canvas")
|
||||
insert("INSERT INTO user_canvas_version (id, user_canvas_id) VALUES (?, ?)", "agent-version", "agent")
|
||||
insert("INSERT INTO api_4_conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "agent-conversation", "agent", "external-user")
|
||||
insert("INSERT INTO chat_channel (id, tenant_id, name, channel, config) VALUES (?, ?, ?, ?, ?)", "channel", "user-1", "Channel", "telegram", "{}")
|
||||
insert("INSERT INTO memory (id, tenant_id, name, embd_id, llm_id, permissions, forgetting_policy) VALUES (?, ?, ?, ?, ?, ?, ?)", "memory", "user-1", "Memory", "embd", "llm", "me", "FIFO")
|
||||
insert("INSERT INTO tenant_model_provider (id, provider_name, tenant_id) VALUES (?, ?, ?)", "provider", "custom", "user-1")
|
||||
insert("INSERT INTO tenant_model_instance (id, instance_name, provider_id, api_key) VALUES (?, ?, ?, ?)", "instance", "custom", "provider", "secret")
|
||||
insert("INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type) VALUES (?, ?, ?, ?, ?)", "model", "model", "provider", "instance", 1)
|
||||
insert("INSERT INTO tenant_model_group (id, group_type, strategy) VALUES (?, ?, ?)", "group", "chat", "weighted")
|
||||
insert("INSERT INTO tenant_model_group_mapping (group_id, provider_id, instance_id, model_id) VALUES (?, ?, ?, ?)", "group", "provider", "instance", "model")
|
||||
|
||||
docEngine := &deletionEngine{}
|
||||
store := &deletionStorage{}
|
||||
core, logs := observer.New(zapcore.InfoLevel)
|
||||
oldLogger := common.Logger
|
||||
common.Logger = zap.New(core)
|
||||
t.Cleanup(func() { common.Logger = oldLogger })
|
||||
service := NewService()
|
||||
service.deleteEngine = docEngine
|
||||
service.deleteStorage = store
|
||||
result, err := service.DeleteUser(t.Context(), "[email protected]")
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteUser: %v", err)
|
||||
}
|
||||
if result.UserCount != 1 || result.TenantCount != 1 {
|
||||
t.Fatalf("unexpected deletion result: %+v", result)
|
||||
}
|
||||
for _, item := range [][2]string{
|
||||
{"knowledgebase", "own-kb"}, {"document", "joined-doc"}, {"file", "own-file"}, {"file", "joined-file"},
|
||||
{"search", "own-search"}, {"search", "joined-search"}, {"dialog", "own-chat"},
|
||||
{"conversation", "own-conversation"}, {"conversation", "joined-conversation"}, {"api_4_conversation", "own-api-conversation"}, {"api_4_conversation", "agent-conversation"},
|
||||
{"user_canvas", "agent"}, {"user_canvas_version", "agent-version"}, {"chat_channel", "channel"},
|
||||
{"memory", "memory"}, {"tenant_model_provider", "provider"}, {"tenant_model_instance", "instance"},
|
||||
{"tenant_model", "model"}, {"tenant_model_group", "group"},
|
||||
} {
|
||||
var count int64
|
||||
if err := db.Table(item[0]).Where("id = ?", item[1]).Count(&count).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("%s retained deleted resource %s", item[0], item[1])
|
||||
}
|
||||
}
|
||||
for _, table := range []string{"conversation_message", "conversation_reference", "api_4_conversation_message", "api_4_conversation_reference", "file2document", "tenant_model_group_mapping"} {
|
||||
var count int64
|
||||
want := int64(0)
|
||||
if table == "file2document" {
|
||||
want = 1
|
||||
}
|
||||
if err := db.Table(table).Count(&count).Error; err != nil || count != want {
|
||||
t.Fatalf("dependent rows remain in %s: count=%d, err=%v", table, count, err)
|
||||
}
|
||||
}
|
||||
for _, tableID := range [][2]string{{"document", "other-doc"}, {"file", "other-file"}, {"search", "other-search"}, {"knowledgebase", "team-kb"}, {"dialog", "team-chat"}, {"conversation", "other-conversation"}} {
|
||||
var count int64
|
||||
if err := db.Table(tableID[0]).Where("id = ?", tableID[1]).Count(&count).Error; err != nil || count != 1 {
|
||||
t.Fatalf("shared resource %s/%s missing: count=%d, err=%v", tableID[0], tableID[1], count, err)
|
||||
}
|
||||
}
|
||||
var team entity.Knowledgebase
|
||||
if err := db.First(&team, "id = ?", "team-kb").Error; err != nil || team.DocNum != 1 || team.TokenNum != 3 || team.ChunkNum != 4 {
|
||||
t.Fatalf("joined dataset counters = %+v, err=%v", team, err)
|
||||
}
|
||||
if !slices.Equal(store.buckets, []string{"own-kb", "own-folder"}) || !slices.Equal(store.files, []string{"own-kb/own.pdf", "team-kb/joined.pdf", "own-folder/own.pdf", "team-folder/joined.pdf"}) {
|
||||
t.Fatalf("storage cleanup = buckets %v, files %v", store.buckets, store.files)
|
||||
}
|
||||
if !slices.Contains(docEngine.dropped, "ragflow_user-1") || !slices.Contains(docEngine.dropped, "memory_user-1") || !slices.Contains(docEngine.deleted, "doc_id:[joined-doc]") || !slices.Contains(docEngine.deleted, "metadata:id:[joined-doc]") {
|
||||
t.Fatalf("index cleanup = dropped %v, deleted %v", docEngine.dropped, docEngine.deleted)
|
||||
}
|
||||
for _, check := range []struct {
|
||||
message string
|
||||
field string
|
||||
want string
|
||||
}{
|
||||
{"Removed document object", "document", "Joined document (joined-doc)"},
|
||||
{"Removed document object", "dataset", "Team (team-kb)"},
|
||||
{"Removed file object", "file", "own.pdf (own-file)"},
|
||||
{"Removed empty folder bucket", "folder", "/ (own-folder)"},
|
||||
} {
|
||||
matched := false
|
||||
for _, entry := range logs.FilterMessage(check.message).All() {
|
||||
if entry.ContextMap()[check.field] == check.want {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
t.Errorf("missing %s log with %s=%q", check.message, check.field, check.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteUserKeepsUserWhenExternalCleanupFails(t *testing.T) {
|
||||
db := setupUserDeletionDB(t)
|
||||
if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id) VALUES (?, ?, ?, ?, ?, ?, ?)", "dataset", "team", "Dataset", "embd", "me", "user-1", "general").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "doc", "dataset", "general", "{}", "pdf", "user-1", "pdf", "doc.pdf").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service := NewService()
|
||||
service.deleteEngine = &deletionEngine{}
|
||||
service.deleteStorage = &deletionStorage{err: errors.New("storage unavailable")}
|
||||
if _, err := service.DeleteUser(t.Context(), "[email protected]"); err == nil || !strings.Contains(err.Error(), "storage unavailable") {
|
||||
t.Fatalf("DeleteUser error = %v", err)
|
||||
}
|
||||
var count int64
|
||||
if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 1 {
|
||||
t.Fatalf("user was deleted after external failure: count=%d, err=%v", count, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteUserContinuesWhenEmptyBucketRemovalFails(t *testing.T) {
|
||||
db := setupUserDeletionDB(t)
|
||||
if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id) VALUES (?, ?, ?, ?, ?, ?, ?)", "dataset", "team", "Dataset", "embd", "me", "user-1", "general").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service := NewService()
|
||||
service.deleteEngine = &deletionEngine{}
|
||||
service.deleteStorage = &deletionStorage{bucketErr: errors.New("bucket is not empty")}
|
||||
if _, err := service.DeleteUser(t.Context(), "[email protected]"); err != nil {
|
||||
t.Fatalf("DeleteUser after bucket removal failure: %v", err)
|
||||
}
|
||||
var count int64
|
||||
if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 0 {
|
||||
t.Fatalf("user retained after bucket failure: count=%d, err=%v", count, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteUserContinuesWhenStorageItemsAreMissing(t *testing.T) {
|
||||
db := setupUserDeletionDB(t)
|
||||
if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id) VALUES (?, ?, ?, ?, ?, ?, ?)", "dataset", "team", "Dataset", "embd", "me", "user-1", "general").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "doc", "dataset", "general", "{}", "pdf", "user-1", "pdf", "doc.pdf").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := &deletionStorage{missing: true}
|
||||
service := NewService()
|
||||
service.deleteEngine = &deletionEngine{}
|
||||
service.deleteStorage = store
|
||||
if _, err := service.DeleteUser(t.Context(), "[email protected]"); err != nil {
|
||||
t.Fatalf("DeleteUser with missing storage items: %v", err)
|
||||
}
|
||||
if len(store.files) != 0 || len(store.buckets) != 0 {
|
||||
t.Fatalf("storage deletion attempted for missing items: files=%v buckets=%v", store.files, store.buckets)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteUserStopsOnStorageCheckError(t *testing.T) {
|
||||
db := setupUserDeletionDB(t)
|
||||
if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id) VALUES (?, ?, ?, ?, ?, ?, ?)", "dataset", "team", "Dataset", "embd", "me", "user-1", "general").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "doc", "dataset", "general", "{}", "pdf", "user-1", "pdf", "doc.pdf").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service := NewService()
|
||||
service.deleteEngine = &deletionEngine{}
|
||||
service.deleteStorage = &deletionStorage{checkErr: errors.New("storage unavailable")}
|
||||
if _, err := service.DeleteUser(t.Context(), "[email protected]"); err == nil || !strings.Contains(err.Error(), "storage unavailable") {
|
||||
t.Fatalf("DeleteUser error = %v", err)
|
||||
}
|
||||
var count int64
|
||||
if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 1 {
|
||||
t.Fatalf("user deleted after storage check failed: count=%d, err=%v", count, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteUserRollsBackDatabaseCleanup(t *testing.T) {
|
||||
db := setupUserDeletionDB(t)
|
||||
if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec("INSERT INTO user_canvas (id, user_id, tags, canvas_category) VALUES (?, ?, ?, ?)", "agent", "user-1", "", "agent_canvas").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec("INSERT INTO user_canvas_version (id, user_canvas_id) VALUES (?, ?)", "version", "agent").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Migrator().DropTable(&entity.Search{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := NewService().DeleteUser(t.Context(), "[email protected]"); err == nil {
|
||||
t.Fatal("expected SQL cleanup failure")
|
||||
}
|
||||
for _, item := range [][2]string{{"user", "user-1"}, {"user_canvas", "agent"}, {"user_canvas_version", "version"}} {
|
||||
var count int64
|
||||
if err := db.Table(item[0]).Where("id = ?", item[1]).Count(&count).Error; err != nil || count != 1 {
|
||||
t.Fatalf("transaction did not preserve %s/%s: count=%d, err=%v", item[0], item[1], count, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1339,8 +1339,15 @@ func (s *chunkImageStorage) ListObjects(ctx context.Context, bucket string, tena
|
||||
func (s *chunkImageStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
func (s *chunkImageStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
|
||||
func (s *chunkImageStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
|
||||
func (s *chunkImageStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
|
||||
func (s *chunkImageStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
|
||||
func (s *chunkImageStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error { return nil }
|
||||
func (s *chunkImageStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
|
||||
return s.exists, nil
|
||||
}
|
||||
func (s *chunkImageStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
func (s *chunkImageStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -318,19 +318,17 @@ func (d *DatasetService) DeleteDatasets(ctx context.Context, ids []string, delet
|
||||
}
|
||||
|
||||
successCount := 0
|
||||
errorsList := make([]string, 0)
|
||||
for _, kb := range kbs {
|
||||
if err := d.deleteDataset(ctx, tenantID, kb); err != nil {
|
||||
errorsList = append(errorsList, err.Error())
|
||||
common.Warn("deleteDataset failed", zap.String("kb_id", kb.ID), zap.Error(err))
|
||||
continue
|
||||
common.Warn("deleteDataset failed", zap.String("dataset", datasetNameAndID(kb)), zap.String("kb_id", kb.ID), zap.Error(err))
|
||||
return nil, common.CodeServerError, err
|
||||
}
|
||||
successCount++
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"success_count": successCount,
|
||||
"errors": errorsList,
|
||||
"errors": []string{},
|
||||
}, common.CodeSuccess, nil
|
||||
}
|
||||
|
||||
@@ -339,6 +337,7 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
|
||||
if storageImpl == nil {
|
||||
return fmt.Errorf("storage not initialized")
|
||||
}
|
||||
dataset := datasetNameAndID(kb)
|
||||
|
||||
// Collect document IDs first so engine cleanup can run before the
|
||||
// transaction (engine ops are not transactional).
|
||||
@@ -346,6 +345,38 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
|
||||
if err := dao.DB.Where("kb_id = ?", kb.ID).Find(&documents).Error; err != nil {
|
||||
return fmt.Errorf("delete dataset error for %s", kb.ID)
|
||||
}
|
||||
cleanupCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
for _, document := range documents {
|
||||
if document.Location == nil || *document.Location == "" {
|
||||
continue
|
||||
}
|
||||
exists, err := storageImpl.ObjectExists(cleanupCtx, kb.ID, *document.Location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check document %s in dataset %s: %w", document.ID, kb.ID, err)
|
||||
}
|
||||
if !exists {
|
||||
common.Warn("Dataset document object already missing", zap.String("document", documentNameAndID(document)), zap.String("document_id", document.ID), zap.String("dataset", dataset), zap.String("bucket", kb.ID))
|
||||
continue
|
||||
}
|
||||
if err := storageImpl.Remove(cleanupCtx, kb.ID, *document.Location); err != nil {
|
||||
return fmt.Errorf("remove document %s from dataset %s: %w", document.ID, kb.ID, err)
|
||||
}
|
||||
common.Info("Removed dataset document object", zap.String("document", documentNameAndID(document)), zap.String("document_id", document.ID), zap.String("dataset", dataset), zap.String("bucket", kb.ID))
|
||||
}
|
||||
exists, err := storageImpl.BucketExistsWithError(cleanupCtx, kb.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check dataset bucket %s: %w", kb.ID, err)
|
||||
}
|
||||
if !exists {
|
||||
common.Warn("Dataset bucket already missing", zap.String("dataset", dataset), zap.String("bucket", kb.ID))
|
||||
} else {
|
||||
if err := storageImpl.RemoveEmptyBucket(cleanupCtx, kb.ID); err != nil {
|
||||
common.Warn("Unable to remove empty dataset bucket", zap.String("dataset", dataset), zap.String("bucket", kb.ID), zap.Error(err))
|
||||
} else {
|
||||
common.Info("Removed empty dataset bucket", zap.String("dataset", dataset), zap.String("bucket", kb.ID))
|
||||
}
|
||||
}
|
||||
docIDs := extractDocIDs(documents)
|
||||
if len(docIDs) > 0 {
|
||||
d.deleteDatasetEngineData(ctx, kb, docIDs)
|
||||
@@ -397,15 +428,24 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
// Dataset uploads and generated objects are stored under the KB ID.
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Minute)
|
||||
defer cancel()
|
||||
if err := storageImpl.RemoveBucket(cleanupCtx, kb.ID); err != nil {
|
||||
return fmt.Errorf("dataset %s deleted but storage cleanup failed: %w", kb.ID, err)
|
||||
}
|
||||
common.Info("Deleted dataset", zap.String("dataset", dataset), zap.String("kb_id", kb.ID))
|
||||
return nil
|
||||
}
|
||||
|
||||
func documentNameAndID(document entity.Document) string {
|
||||
if document.Name == nil || *document.Name == "" {
|
||||
return document.ID
|
||||
}
|
||||
return fmt.Sprintf("%s (%s)", *document.Name, document.ID)
|
||||
}
|
||||
|
||||
func datasetNameAndID(kb *entity.Knowledgebase) string {
|
||||
if kb.Name == "" {
|
||||
return kb.ID
|
||||
}
|
||||
return fmt.Sprintf("%s (%s)", kb.Name, kb.ID)
|
||||
}
|
||||
|
||||
func (d *DatasetService) ListDatasets(ctx context.Context, id, name string, page, pageSize int, terms []dao.OrderTerm, keywords string, ownerIDs []string, parserID, userID string, ids []string) ([]map[string]interface{}, int64, common.ErrorCode, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id != "" && len(ids) > 0 {
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package dataset
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/storage"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
"go.uber.org/zap/zaptest/observer"
|
||||
)
|
||||
|
||||
func TestDeleteDatasetLeavesNonemptyBucket(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Dataset")
|
||||
if err := db.Exec("INSERT INTO document (id, kb_id, name, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", "doc-1", "kb-1", "Document", "general", "{}", "pdf", "tenant-1", "pdf", "tracked").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := storage.NewMemoryStorage()
|
||||
if err := store.Put(t.Context(), "kb-1", "tracked", []byte("content")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Put(t.Context(), "kb-1", "untracked", []byte("content")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
factory := storage.GetStorageFactory()
|
||||
previous := factory.GetStorage()
|
||||
factory.SetStorage(store)
|
||||
t.Cleanup(func() { factory.SetStorage(previous) })
|
||||
core, logs := observer.New(zapcore.InfoLevel)
|
||||
previousLogger := common.Logger
|
||||
common.Logger = zap.New(core)
|
||||
t.Cleanup(func() { common.Logger = previousLogger })
|
||||
|
||||
kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service := testDatasetUpdateService(t)
|
||||
if err := service.deleteDataset(t.Context(), "tenant-1", kb); err != nil {
|
||||
t.Fatalf("delete dataset with nonempty bucket: %v", err)
|
||||
}
|
||||
if !store.ObjExist(t.Context(), "kb-1", "untracked") {
|
||||
t.Fatal("dataset deletion removed an untracked object")
|
||||
}
|
||||
if store.ObjExist(t.Context(), "kb-1", "tracked") {
|
||||
t.Fatal("dataset deletion retained a document object")
|
||||
}
|
||||
var count int64
|
||||
if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Count(&count).Error; err != nil || count != 0 {
|
||||
t.Fatalf("dataset retained after storage failure: count=%d, err=%v", count, err)
|
||||
}
|
||||
if entries := logs.FilterMessage("Removed dataset document object").All(); len(entries) != 1 || entries[0].ContextMap()["document"] != "Document (doc-1)" || entries[0].ContextMap()["dataset"] != "Dataset (kb-1)" {
|
||||
t.Errorf("document deletion logs = %v", entries)
|
||||
}
|
||||
if entries := logs.FilterMessage("Deleted dataset").All(); len(entries) != 1 || entries[0].ContextMap()["dataset"] != "Dataset (kb-1)" {
|
||||
t.Errorf("dataset deletion logs = %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteDatasetContinuesWhenStorageIsMissing(t *testing.T) {
|
||||
db := setupServiceTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Dataset")
|
||||
if err := db.Exec("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "doc-1", "kb-1", "general", "{}", "pdf", "tenant-1", "pdf", "missing").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
factory := storage.GetStorageFactory()
|
||||
previous := factory.GetStorage()
|
||||
factory.SetStorage(storage.NewMemoryStorage())
|
||||
t.Cleanup(func() { factory.SetStorage(previous) })
|
||||
kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDatasetUpdateService(t).deleteDataset(t.Context(), "tenant-1", kb); err != nil {
|
||||
t.Fatalf("delete dataset with missing storage: %v", err)
|
||||
}
|
||||
var count int64
|
||||
if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Count(&count).Error; err != nil || count != 0 {
|
||||
t.Fatalf("dataset retained after missing storage: count=%d, err=%v", count, err)
|
||||
}
|
||||
}
|
||||
@@ -123,8 +123,15 @@ func (f *fakeUploadStorage) ListObjects(ctx context.Context, bucket string, tena
|
||||
func (f *fakeUploadStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
func (f *fakeUploadStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
|
||||
func (f *fakeUploadStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
|
||||
func (f *fakeUploadStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
|
||||
func (f *fakeUploadStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
|
||||
func (f *fakeUploadStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error { return nil }
|
||||
func (f *fakeUploadStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
|
||||
return f.ObjExist(ctx, bucket, fnm), nil
|
||||
}
|
||||
func (f *fakeUploadStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
func (f *fakeUploadStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
|
||||
v, ok := f.objects[f.key(srcBucket, srcPath)]
|
||||
if !ok {
|
||||
|
||||
@@ -2,12 +2,14 @@ package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/storage"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// DeleteFiles deletes files by IDs
|
||||
@@ -58,17 +60,36 @@ func (s *FileService) DeleteFiles(ctx context.Context, uid string, fileIDs []str
|
||||
// deleteSingleFile deletes a single file (not folder)
|
||||
// Matches Python's _delete_single_file function
|
||||
func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) error {
|
||||
// 1. Delete storage object
|
||||
if file.Location != nil && *file.Location != "" {
|
||||
storageImpl := storage.GetStorageFactory().GetStorage()
|
||||
if storageImpl != nil {
|
||||
if err := storageImpl.Remove(ctx, file.ParentID, *file.Location); err != nil {
|
||||
common.Logger.Error(fmt.Sprintf("Fail to remove object: %s/%s, error: %v", file.ParentID, *file.Location, err))
|
||||
}
|
||||
}
|
||||
storageImpl := storage.GetStorageFactory().GetStorage()
|
||||
if err := removeFileObject(ctx, storageImpl, file); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.deleteSingleFileRecords(ctx, file)
|
||||
}
|
||||
|
||||
// 2. Handle associated documents
|
||||
func removeFileObject(ctx context.Context, storageImpl storage.Storage, file *entity.File) error {
|
||||
if file.Location != nil && *file.Location != "" {
|
||||
if storageImpl == nil {
|
||||
return fmt.Errorf("storage is not configured for file %s", file.ID)
|
||||
}
|
||||
exists, err := storageImpl.ObjectExists(ctx, file.ParentID, *file.Location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check file %s: %w", file.ID, err)
|
||||
}
|
||||
if !exists {
|
||||
common.Warn("File object already missing", zap.String("file", fileNameAndID(file)), zap.String("file_id", file.ID), zap.String("bucket", file.ParentID))
|
||||
return nil
|
||||
}
|
||||
if err := storageImpl.Remove(ctx, file.ParentID, *file.Location); err != nil {
|
||||
return fmt.Errorf("remove file %s: %w", file.ID, err)
|
||||
}
|
||||
common.Info("Removed file object", zap.String("file", fileNameAndID(file)), zap.String("file_id", file.ID), zap.String("bucket", file.ParentID))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *FileService) deleteSingleFileRecords(ctx context.Context, file *entity.File) error {
|
||||
// Handle associated documents
|
||||
informs, err := s.file2DocumentDAO.GetByFileID(ctx, dao.DB, file.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get file2document mappings: %w", err)
|
||||
@@ -81,10 +102,7 @@ func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) e
|
||||
docID := *inform.DocumentID
|
||||
if s.documentService != nil {
|
||||
if err = s.documentService.RemoveDocumentKeepFile(ctx, docID); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return fmt.Errorf("context cancelled while removing document %s: %w", docID, err)
|
||||
}
|
||||
common.Logger.Error(fmt.Sprintf("Fail to remove document: %s, error: %v", docID, err))
|
||||
return fmt.Errorf("remove document %s: %w", docID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -95,10 +113,11 @@ func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) e
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Delete file record
|
||||
// Delete file record
|
||||
if err = s.fileDAO.Delete(ctx, dao.DB, file.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
common.Info("Deleted file", zap.String("file", fileNameAndID(file)), zap.String("file_id", file.ID))
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -106,30 +125,72 @@ func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) e
|
||||
// deleteFolderRecursive recursively deletes a folder and its contents
|
||||
// Matches Python's _delete_folder_recursive function
|
||||
func (s *FileService) deleteFolderRecursive(ctx context.Context, folder *entity.File, uid string) error {
|
||||
// Get all sub-files
|
||||
storageImpl := storage.GetStorageFactory().GetStorage()
|
||||
if storageImpl == nil {
|
||||
return fmt.Errorf("storage is not configured for folder %s", folder.ID)
|
||||
}
|
||||
if err := s.removeFolderObjectsRecursive(ctx, folder, storageImpl); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.deleteFolderRecordsRecursive(ctx, folder)
|
||||
}
|
||||
|
||||
func (s *FileService) removeFolderObjectsRecursive(ctx context.Context, folder *entity.File, storageImpl storage.Storage) error {
|
||||
subFiles, err := s.fileDAO.ListByParentID(ctx, dao.DB, folder.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, subFile := range subFiles {
|
||||
if subFile.Type == FileTypeFolder {
|
||||
// Recursively delete subfolder
|
||||
if err = s.deleteFolderRecursive(ctx, subFile, uid); err != nil {
|
||||
if err = s.removeFolderObjectsRecursive(ctx, subFile, storageImpl); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// Delete single file
|
||||
if err = s.deleteSingleFile(ctx, subFile); err != nil {
|
||||
if err = removeFileObject(ctx, storageImpl, subFile); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
exists, err := storageImpl.BucketExistsWithError(ctx, folder.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check folder bucket %s: %w", folder.ID, err)
|
||||
}
|
||||
if !exists {
|
||||
common.Warn("Folder bucket already missing", zap.String("folder", fileNameAndID(folder)), zap.String("bucket", folder.ID))
|
||||
return nil
|
||||
}
|
||||
if err := storageImpl.RemoveEmptyBucket(ctx, folder.ID); err != nil {
|
||||
common.Warn("Unable to remove empty folder bucket", zap.String("folder", fileNameAndID(folder)), zap.String("bucket", folder.ID), zap.Error(err))
|
||||
} else {
|
||||
common.Info("Removed empty folder bucket", zap.String("folder", fileNameAndID(folder)), zap.String("bucket", folder.ID))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete the folder itself
|
||||
func (s *FileService) deleteFolderRecordsRecursive(ctx context.Context, folder *entity.File) error {
|
||||
subFiles, err := s.fileDAO.ListByParentID(ctx, dao.DB, folder.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, subFile := range subFiles {
|
||||
if subFile.Type == FileTypeFolder {
|
||||
if err = s.deleteFolderRecordsRecursive(ctx, subFile); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err = s.deleteSingleFileRecords(ctx, subFile); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err = s.fileDAO.Delete(ctx, dao.DB, folder.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
common.Info("Deleted folder", zap.String("folder", fileNameAndID(folder)), zap.String("folder_id", folder.ID))
|
||||
return nil
|
||||
}
|
||||
|
||||
func fileNameAndID(file *entity.File) string {
|
||||
if file.Name == "" {
|
||||
return file.ID
|
||||
}
|
||||
return fmt.Sprintf("%s (%s)", file.Name, file.ID)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/storage"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
"go.uber.org/zap/zaptest/observer"
|
||||
)
|
||||
|
||||
type failingObjectCheckStorage struct{ storage.Storage }
|
||||
|
||||
func (f failingObjectCheckStorage) ObjectExists(context.Context, string, string) (bool, error) {
|
||||
return false, errors.New("storage unavailable")
|
||||
}
|
||||
|
||||
func TestDeleteFolderLeavesNonemptyBucket(t *testing.T) {
|
||||
db := setupFolderTestDB(t)
|
||||
folder, err := testFileService().fileDAO.CreateFolder(t.Context(), db, "root", "tenant-1", "folder", FileTypeFolder)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
insertFolderTestFile(t, "file-1", folder.ID, "tracked")
|
||||
store := storage.NewMemoryStorage()
|
||||
if err := store.Put(t.Context(), folder.ID, "tracked", []byte("content")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Put(t.Context(), folder.ID, "untracked", []byte("content")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
factory := storage.GetStorageFactory()
|
||||
previous := factory.GetStorage()
|
||||
factory.SetStorage(store)
|
||||
t.Cleanup(func() { factory.SetStorage(previous) })
|
||||
core, logs := observer.New(zapcore.InfoLevel)
|
||||
previousLogger := common.Logger
|
||||
common.Logger = zap.New(core)
|
||||
t.Cleanup(func() { common.Logger = previousLogger })
|
||||
|
||||
service := testFileService()
|
||||
if err := service.deleteFolderRecursive(t.Context(), folder, "tenant-1"); err != nil {
|
||||
t.Fatalf("delete folder with nonempty bucket: %v", err)
|
||||
}
|
||||
if !store.ObjExist(t.Context(), folder.ID, "untracked") {
|
||||
t.Fatal("folder deletion removed an untracked object")
|
||||
}
|
||||
if store.ObjExist(t.Context(), folder.ID, "tracked") {
|
||||
t.Fatal("folder deletion retained a file object")
|
||||
}
|
||||
if _, err := service.fileDAO.GetByID(t.Context(), db, folder.ID); err == nil {
|
||||
t.Fatal("folder record retained after storage failure")
|
||||
}
|
||||
if entries := logs.FilterMessage("Deleted file").All(); len(entries) != 1 || entries[0].ContextMap()["file"] != "tracked (file-1)" {
|
||||
t.Errorf("file deletion logs = %v", entries)
|
||||
}
|
||||
if entries := logs.FilterMessage("Deleted folder").All(); len(entries) != 1 || entries[0].ContextMap()["folder"] != "folder ("+folder.ID+")" {
|
||||
t.Errorf("folder deletion logs = %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteFolderStopsOnStorageCheckError(t *testing.T) {
|
||||
db := setupFolderTestDB(t)
|
||||
folder, err := testFileService().fileDAO.CreateFolder(t.Context(), db, "root", "tenant-1", "folder", FileTypeFolder)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
insertFolderTestFile(t, "file-1", folder.ID, "tracked")
|
||||
factory := storage.GetStorageFactory()
|
||||
previous := factory.GetStorage()
|
||||
factory.SetStorage(failingObjectCheckStorage{storage.NewMemoryStorage()})
|
||||
t.Cleanup(func() { factory.SetStorage(previous) })
|
||||
if err := testFileService().deleteFolderRecursive(t.Context(), folder, "tenant-1"); err == nil {
|
||||
t.Fatal("expected storage check error")
|
||||
}
|
||||
if _, err := testFileService().fileDAO.GetByID(t.Context(), db, "file-1"); err != nil {
|
||||
t.Fatalf("file record deleted after storage check failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteFolderContinuesWhenStorageIsMissing(t *testing.T) {
|
||||
db := setupFolderTestDB(t)
|
||||
service := testFileService()
|
||||
folder, err := service.fileDAO.CreateFolder(t.Context(), db, "root", "tenant-1", "folder", FileTypeFolder)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
insertFolderTestFile(t, "file-1", folder.ID, "missing")
|
||||
factory := storage.GetStorageFactory()
|
||||
previous := factory.GetStorage()
|
||||
factory.SetStorage(storage.NewMemoryStorage())
|
||||
t.Cleanup(func() { factory.SetStorage(previous) })
|
||||
if err := service.deleteFolderRecursive(t.Context(), folder, "tenant-1"); err != nil {
|
||||
t.Fatalf("delete folder with missing storage: %v", err)
|
||||
}
|
||||
if _, err := service.fileDAO.GetByID(t.Context(), db, folder.ID); err == nil {
|
||||
t.Fatal("folder record retained after missing storage")
|
||||
}
|
||||
}
|
||||
@@ -123,6 +123,16 @@ func (f *fakeStorage) RemoveBucket(ctx context.Context, bucket string) error {
|
||||
panic("not implemented in fakeStorage")
|
||||
}
|
||||
|
||||
func (f *fakeStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
|
||||
panic("not implemented in fakeStorage")
|
||||
}
|
||||
func (f *fakeStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
|
||||
return f.ObjExist(ctx, bucket, fnm), nil
|
||||
}
|
||||
func (f *fakeStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
|
||||
panic("not implemented in fakeStorage")
|
||||
}
|
||||
|
||||
func (f *fakeStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
|
||||
panic("not implemented in fakeStorage")
|
||||
}
|
||||
|
||||
@@ -922,7 +922,7 @@ func (s *MemoryService) DeleteMemory(ctx context.Context, userID, memoryID strin
|
||||
|
||||
// TODO: Delete associated message index - Implementation pending MessageService
|
||||
if s.docEngine != nil && engine.IsOceanBaseFamily(s.docEngine.GetType()) {
|
||||
if err := s.docEngine.DropChunkStore(ctx, memoryIndexName(memory.TenantID), memoryID); err != nil {
|
||||
if err := s.docEngine.DropChunkStore(ctx, MemoryIndexName(memory.TenantID), memoryID); err != nil {
|
||||
return fmt.Errorf("delete memory messages: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -964,7 +964,7 @@ func (s *MemoryService) ForgetMessage(ctx context.Context, userID string, memory
|
||||
condition := map[string]interface{}{
|
||||
"id": messageDocID,
|
||||
}
|
||||
indexName := memoryIndexName(memory.TenantID)
|
||||
indexName := MemoryIndexName(memory.TenantID)
|
||||
|
||||
if err = s.docEngine.UpdateChunks(ctx, condition, updates, indexName, memoryID); err != nil {
|
||||
if isMessageDocumentNotFound(err) {
|
||||
@@ -1097,7 +1097,7 @@ func (s *MemoryService) UpdateMessageStatus(ctx context.Context, userID, memoryI
|
||||
condition := map[string]interface{}{
|
||||
"id": messageDocID,
|
||||
}
|
||||
indexName := memoryIndexName(memory.TenantID)
|
||||
indexName := MemoryIndexName(memory.TenantID)
|
||||
if err = s.docEngine.UpdateChunks(ctx, condition, updates, indexName, memoryID); err != nil {
|
||||
if isMessageDocumentNotFound(err) {
|
||||
return false, &ResourceNotFoundError{Resource: "Message", ID: messageDocID}
|
||||
@@ -1121,7 +1121,7 @@ func (s *MemoryService) GetMessageContent(ctx context.Context, userID, memoryID
|
||||
return nil, errors.New("message store is not initialized")
|
||||
}
|
||||
|
||||
indexName := memoryIndexName(memory.TenantID)
|
||||
indexName := MemoryIndexName(memory.TenantID)
|
||||
docID := fmt.Sprintf("%s_%d", memoryID, messageID)
|
||||
res, err := s.docEngine.GetChunk(ctx, indexName, docID, []string{memoryID})
|
||||
if err != nil {
|
||||
@@ -1423,7 +1423,8 @@ func memoryMessageSelectFields() []string {
|
||||
}
|
||||
}
|
||||
|
||||
func memoryIndexName(tenantID string) string {
|
||||
// MemoryIndexName returns the message index shared by a tenant's memories.
|
||||
func MemoryIndexName(tenantID string) string {
|
||||
prefix := strings.TrimSpace(common.GetEnv(common.EnvESIndexPrefix))
|
||||
if prefix == "" {
|
||||
return fmt.Sprintf("memory_%s", tenantID)
|
||||
@@ -1438,7 +1439,7 @@ func memorySearchIndexNames(memories []*entity.Memory) []string {
|
||||
if memory == nil {
|
||||
continue
|
||||
}
|
||||
indexName := memoryIndexName(memory.TenantID)
|
||||
indexName := MemoryIndexName(memory.TenantID)
|
||||
if engine.GetEngineType() == "infinity" {
|
||||
indexName = fmt.Sprintf("%s_%s", indexName, memory.ID)
|
||||
}
|
||||
|
||||
@@ -353,7 +353,7 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Cr
|
||||
message["doc_id"] = message["memory_id"]
|
||||
}
|
||||
|
||||
indexName := memoryIndexName(mem.TenantID)
|
||||
indexName := MemoryIndexName(mem.TenantID)
|
||||
exists, err := s.memories.docEngine.ChunkStoreExists(ctx, indexName, mem.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check message index: %w", err)
|
||||
|
||||
@@ -34,12 +34,12 @@ func TestIsMessageDocumentNotFound(t *testing.T) {
|
||||
|
||||
func TestMemoryIndexNameMatchesPythonPrefix(t *testing.T) {
|
||||
t.Setenv(common.EnvESIndexPrefix, "")
|
||||
if got := memoryIndexName("tenant-1"); got != "memory_tenant-1" {
|
||||
t.Fatalf("memoryIndexName() = %q", got)
|
||||
if got := MemoryIndexName("tenant-1"); got != "memory_tenant-1" {
|
||||
t.Fatalf("MemoryIndexName() = %q", got)
|
||||
}
|
||||
t.Setenv(common.EnvESIndexPrefix, "legacy")
|
||||
if got := memoryIndexName("tenant-1"); got != "memory_legacy_tenant-1" {
|
||||
t.Fatalf("memoryIndexName() with prefix = %q", got)
|
||||
if got := MemoryIndexName("tenant-1"); got != "memory_legacy_tenant-1" {
|
||||
t.Fatalf("MemoryIndexName() with prefix = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -168,7 +168,7 @@ func (s *SkillIndexerService) IndexSkill(ctx context.Context, tenantID, spaceID
|
||||
}
|
||||
}
|
||||
|
||||
indexName := getSkillIndexName(tenantID, spaceID)
|
||||
indexName := SkillIndexName(tenantID, spaceID)
|
||||
|
||||
// For Infinity: ensure table exists with correct dimension BEFORE inserting
|
||||
if docEngine.GetType() == "infinity" {
|
||||
@@ -249,7 +249,7 @@ func (s *SkillIndexerService) BatchIndexSkills(ctx context.Context, tenantID, sp
|
||||
}
|
||||
|
||||
// Ensure index exists with correct dimension
|
||||
indexName := getSkillIndexName(tenantID, spaceID)
|
||||
indexName := SkillIndexName(tenantID, spaceID)
|
||||
if docEngine.GetType() == "infinity" {
|
||||
// For Infinity: must ensure table exists with correct dimension BEFORE inserting
|
||||
common.Info(fmt.Sprintf("Checking if index exists: %s", indexName))
|
||||
@@ -362,7 +362,7 @@ func (s *SkillIndexerService) BatchIndexSkills(ctx context.Context, tenantID, sp
|
||||
// Returns nil if the document doesn't exist (idempotent delete)
|
||||
func (s *SkillIndexerService) DeleteSkillIndex(ctx context.Context, tenantID, spaceID, skillID string, docEngine engine.DocEngine) error {
|
||||
spaceID = normalizeSpaceID(spaceID)
|
||||
indexName := getSkillIndexName(tenantID, spaceID)
|
||||
indexName := SkillIndexName(tenantID, spaceID)
|
||||
// ES document ID cannot contain '/' - replace with '_'
|
||||
docID := strings.ReplaceAll(skillID, "/", "_")
|
||||
if err := docEngine.DeleteDocument(ctx, indexName, docID); err != nil {
|
||||
@@ -381,7 +381,7 @@ func (s *SkillIndexerService) DeleteSkillIndex(ctx context.Context, tenantID, sp
|
||||
// Deletes all versions: both new format (skillname) and old format (skillname_x.x.x)
|
||||
func (s *SkillIndexerService) DeleteSkillByName(ctx context.Context, tenantID, spaceID, skillName string, docEngine engine.DocEngine) error {
|
||||
spaceID = normalizeSpaceID(spaceID)
|
||||
indexName := getSkillIndexName(tenantID, spaceID)
|
||||
indexName := SkillIndexName(tenantID, spaceID)
|
||||
|
||||
docID := strings.ReplaceAll(skillName, "/", "_")
|
||||
if err := docEngine.DeleteDocument(ctx, indexName, docID); err != nil {
|
||||
@@ -436,7 +436,7 @@ func (s *SkillIndexerService) ReindexAll(ctx context.Context, tenantID, spaceID
|
||||
common.Info(fmt.Sprintf("ReindexAll: new embedding dimension is %d", newDimension))
|
||||
|
||||
// Delete existing index and recreate with new dimension (for both ES and Infinity)
|
||||
indexName := getSkillIndexName(tenantID, spaceID)
|
||||
indexName := SkillIndexName(tenantID, spaceID)
|
||||
exists, _ := docEngine.ChunkStoreExists(ctx, indexName, "skill")
|
||||
if exists {
|
||||
common.Info(fmt.Sprintf("ReindexAll: deleting existing index %s", indexName))
|
||||
@@ -843,7 +843,7 @@ func (s *SkillIndexerService) cleanupOldVersions(ctx context.Context, tenantID,
|
||||
// InitializeIndex initializes the skill search index for a tenant
|
||||
func (s *SkillIndexerService) InitializeIndex(ctx context.Context, tenantID, spaceID string, docEngine engine.DocEngine, embdID string) error {
|
||||
// Check if index exists
|
||||
indexName := getSkillIndexName(tenantID, spaceID)
|
||||
indexName := SkillIndexName(tenantID, spaceID)
|
||||
|
||||
common.Info("Checking skill index existence", zap.String("indexName", indexName), zap.String("tenantID", tenantID), zap.String("spaceID", spaceID))
|
||||
|
||||
@@ -874,7 +874,7 @@ func (s *SkillIndexerService) createIndex(ctx context.Context, tenantID, spaceID
|
||||
|
||||
// createIndexWithDimension creates the skill index with a specific vector dimension
|
||||
func (s *SkillIndexerService) createIndexWithDimension(ctx context.Context, tenantID, spaceID string, docEngine engine.DocEngine, embdID string, dimension int) error {
|
||||
indexName := getSkillIndexName(tenantID, spaceID)
|
||||
indexName := SkillIndexName(tenantID, spaceID)
|
||||
|
||||
common.Info(fmt.Sprintf("Creating skill index with dimension %d", dimension),
|
||||
zap.String("indexName", indexName),
|
||||
|
||||
@@ -222,7 +222,7 @@ func (s *SkillSearchService) Search(ctx context.Context, req *SearchRequest, doc
|
||||
}
|
||||
|
||||
// Check if index exists before searching
|
||||
indexName := getSkillIndexName(req.TenantID, req.SpaceID)
|
||||
indexName := SkillIndexName(req.TenantID, req.SpaceID)
|
||||
common.Debug("Searching skills", zap.String("indexName", indexName), zap.String("query", req.Query))
|
||||
|
||||
indexExists, err := docEngine.ChunkStoreExists(ctx, indexName, "skill")
|
||||
@@ -694,7 +694,8 @@ func (s *SkillSearchService) getEmbedding(ctx context.Context, text, embdID, ten
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
func getSkillIndexName(tenantID, spaceID string) string {
|
||||
// SkillIndexName returns the index used by a tenant's skill space.
|
||||
func SkillIndexName(tenantID, spaceID string) string {
|
||||
spaceID = normalizeSpaceID(spaceID)
|
||||
spaceID = strings.ToLower(spaceID)
|
||||
replacer := strings.NewReplacer("-", "_", "/", "_", "\\", "_", " ", "_", ".", "_", ":", "_")
|
||||
|
||||
@@ -439,7 +439,7 @@ func (s *SkillSpaceService) asyncDeleteSpace(ctx context.Context, spaceID, folde
|
||||
|
||||
// Step 1: Delete the search index
|
||||
if docEngine != nil {
|
||||
indexName := getSkillIndexName(tenantID, spaceID)
|
||||
indexName := SkillIndexName(tenantID, spaceID)
|
||||
common.Info("Async deleting space index", zap.String("index", indexName), zap.String("spaceID", spaceID))
|
||||
deleteCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
if err := docEngine.DropChunkStore(deleteCtx, indexName, "skill"); err != nil {
|
||||
|
||||
@@ -111,6 +111,9 @@ func (m *GCSStorage) Remove(ctx context.Context, bucketName, objectName string,
|
||||
|
||||
obj := m.client.Bucket(bucketName).Object(objectName)
|
||||
if err := obj.Delete(ctx); err != nil {
|
||||
if errors.Is(err, storage.ErrObjectNotExist) || errors.Is(err, storage.ErrBucketNotExist) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("fail to delete object: %w", err)
|
||||
}
|
||||
|
||||
@@ -130,6 +133,17 @@ func (m *GCSStorage) ObjExist(ctx context.Context, bucketName, objectName string
|
||||
return true
|
||||
}
|
||||
|
||||
func (m *GCSStorage) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, error) {
|
||||
_, err := m.client.Bucket(bucketName).Object(objectName).Attrs(ctx)
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if errors.Is(err, storage.ErrObjectNotExist) || errors.Is(err, storage.ErrBucketNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
func (m *GCSStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
|
||||
|
||||
bucketObject := m.client.Bucket(bucket)
|
||||
@@ -182,6 +196,17 @@ func (m *GCSStorage) BucketExists(ctx context.Context, bucket string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (m *GCSStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
|
||||
_, err := m.client.Bucket(bucket).Attrs(ctx)
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if errors.Is(err, storage.ErrBucketNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
// RemoveBucket removes a bucket and all its objects
|
||||
func (m *GCSStorage) RemoveBucket(ctx context.Context, bucketName string) error {
|
||||
if bucketName == "" {
|
||||
@@ -212,6 +237,18 @@ func (m *GCSStorage) RemoveBucket(ctx context.Context, bucketName string) error
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveEmptyBucket removes a bucket only if GCS finds it empty.
|
||||
func (m *GCSStorage) RemoveEmptyBucket(ctx context.Context, bucketName string) error {
|
||||
if bucketName == "" {
|
||||
return fmt.Errorf("attempt to delete bucket without name")
|
||||
}
|
||||
err := m.client.Bucket(bucketName).Delete(ctx)
|
||||
if errors.Is(err, storage.ErrBucketNotExist) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Copy copies an object from source to destination
|
||||
func (m *GCSStorage) Copy(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) bool {
|
||||
|
||||
|
||||
@@ -137,6 +137,16 @@ func (m *MemoryStorage) ObjExist(ctx context.Context, bucket, fnm string, tenant
|
||||
return ok
|
||||
}
|
||||
|
||||
func (m *MemoryStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
_, exists := m.objects[bucket][fnm]
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (m *MemoryStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
@@ -183,6 +193,13 @@ func (m *MemoryStorage) BucketExists(ctx context.Context, bucket string) bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
func (m *MemoryStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return m.BucketExists(ctx, bucket), nil
|
||||
}
|
||||
|
||||
// RemoveBucket deletes a bucket and all of its keys. Removing a
|
||||
// non-existent bucket is a no-op and returns nil.
|
||||
func (m *MemoryStorage) RemoveBucket(ctx context.Context, bucket string) error {
|
||||
@@ -197,6 +214,20 @@ func (m *MemoryStorage) RemoveBucket(ctx context.Context, bucket string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveEmptyBucket deletes a bucket only when it contains no objects.
|
||||
func (m *MemoryStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if len(m.objects[bucket]) != 0 {
|
||||
return fmt.Errorf("bucket %s is not empty", bucket)
|
||||
}
|
||||
delete(m.objects, bucket)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy duplicates an object from srcBucket/srcKey to destBucket/destKey.
|
||||
// The source is left untouched. Returns false if the source does not exist
|
||||
// or if the destination bucket creation fails.
|
||||
|
||||
@@ -156,6 +156,29 @@ func TestMemoryStorage_RemoveBucket(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStorage_RemoveEmptyBucket(t *testing.T) {
|
||||
ms := newTestMemory(t)
|
||||
ctx := t.Context()
|
||||
if err := ms.Put(ctx, "bucket", "file", []byte("content")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ms.RemoveEmptyBucket(ctx, "bucket"); err == nil {
|
||||
t.Fatal("RemoveEmptyBucket removed a nonempty bucket")
|
||||
}
|
||||
if !ms.ObjExist(ctx, "bucket", "file") {
|
||||
t.Fatal("RemoveEmptyBucket removed the file")
|
||||
}
|
||||
if err := ms.Remove(ctx, "bucket", "file"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ms.RemoveEmptyBucket(ctx, "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ms.BucketExists(ctx, "bucket") {
|
||||
t.Fatal("RemoveEmptyBucket retained an empty bucket")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStorage_CopyMove(t *testing.T) {
|
||||
ms := newTestMemory(t)
|
||||
ctx := t.Context()
|
||||
|
||||
@@ -234,6 +234,10 @@ func (m *MinioStorage) Remove(ctx context.Context, bucket, fnm string, tenantID
|
||||
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
if err := m.client.RemoveObject(ctx, bucket, fnm, minio.RemoveObjectOptions{}); err != nil {
|
||||
code := minio.ToErrorResponse(err).Code
|
||||
if code == "NoSuchKey" || code == "NoSuchBucket" {
|
||||
return nil
|
||||
}
|
||||
common.Warn("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
@@ -263,6 +267,19 @@ func (m *MinioStorage) ObjExist(ctx context.Context, bucket, fnm string, tenantI
|
||||
return true
|
||||
}
|
||||
|
||||
func (m *MinioStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
|
||||
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
|
||||
_, err := m.client.StatObject(ctx, bucket, fnm, minio.StatObjectOptions{})
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
code := minio.ToErrorResponse(err).Code
|
||||
if code == "NoSuchKey" || code == "NoSuchBucket" {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
// GetPresignedURL generates a presigned URL for accessing an object
|
||||
func (m *MinioStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
|
||||
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
|
||||
@@ -303,6 +320,13 @@ func (m *MinioStorage) BucketExists(ctx context.Context, bucket string) bool {
|
||||
return exists
|
||||
}
|
||||
|
||||
func (m *MinioStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
|
||||
if m.bucket != "" {
|
||||
bucket = m.bucket
|
||||
}
|
||||
return m.client.BucketExists(ctx, bucket)
|
||||
}
|
||||
|
||||
func (m *MinioStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
|
||||
|
||||
var objects []string
|
||||
@@ -401,6 +425,41 @@ func (m *MinioStorage) RemoveBucket(ctx context.Context, bucket string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveEmptyBucket removes a bucket only when it contains no object versions.
|
||||
func (m *MinioStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
|
||||
actualBucket := bucket
|
||||
prefix := ""
|
||||
if m.bucket != "" {
|
||||
actualBucket = m.bucket
|
||||
if m.prefixPath != "" {
|
||||
prefix = m.prefixPath + "/"
|
||||
}
|
||||
prefix += bucket + "/"
|
||||
}
|
||||
exists, err := m.client.BucketExists(ctx, actualBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
listCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
for object := range m.client.ListObjects(listCtx, actualBucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true, WithVersions: true}) {
|
||||
if object.Err != nil {
|
||||
return object.Err
|
||||
}
|
||||
return fmt.Errorf("bucket %s is not empty", bucket)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.bucket == "" {
|
||||
return m.client.RemoveBucket(ctx, actualBucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy copies an object from source to destination
|
||||
func (m *MinioStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
|
||||
srcBucket, srcPath = m.resolveBucketAndPath(srcBucket, srcPath)
|
||||
|
||||
+44
-2
@@ -253,6 +253,9 @@ func (o *OSSStorage) Remove(ctx context.Context, bucket, fnm string, tenantID ..
|
||||
Key: aws.String(fnm),
|
||||
})
|
||||
if err != nil {
|
||||
if isS3NotFound(err) {
|
||||
return nil
|
||||
}
|
||||
common.Error("Failed to remove object", err, zap.String("bucket", bucket), zap.String("key", fnm))
|
||||
return err
|
||||
}
|
||||
@@ -278,6 +281,18 @@ func (o *OSSStorage) ObjExist(ctx context.Context, bucket, fnm string, tenantID
|
||||
return true
|
||||
}
|
||||
|
||||
func (o *OSSStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
|
||||
bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
|
||||
_, err := o.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(fnm)})
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if isS3NotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
func (o *OSSStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
|
||||
|
||||
listInput := &s3.ListObjectsV2Input{
|
||||
@@ -344,13 +359,28 @@ func (o *OSSStorage) BucketExists(ctx context.Context, bucket string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (o *OSSStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
|
||||
if o.bucket != "" {
|
||||
bucket = o.bucket
|
||||
}
|
||||
_, err := o.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if isS3BucketNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
// RemoveBucket removes a bucket and all its objects
|
||||
func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {
|
||||
actualBucket := bucket
|
||||
if o.bucket != "" {
|
||||
actualBucket = o.bucket
|
||||
return fmt.Errorf("cannot remove logical OSS bucket %s: shared bucket mode does not isolate objects by logical bucket", bucket)
|
||||
}
|
||||
|
||||
actualBucket := bucket
|
||||
|
||||
// Check if bucket exists
|
||||
if !o.BucketExists(ctx, actualBucket) {
|
||||
return nil
|
||||
@@ -396,6 +426,18 @@ func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveEmptyBucket removes a physical bucket only when it is empty.
|
||||
func (o *OSSStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
|
||||
if o.bucket != "" {
|
||||
return fmt.Errorf("cannot remove logical OSS bucket %s: shared bucket mode does not isolate objects by logical bucket", bucket)
|
||||
}
|
||||
_, err := o.client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)})
|
||||
if isS3NotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Copy copies an object from source to destination
|
||||
func (o *OSSStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
|
||||
srcBucket, srcPath = o.resolveBucketAndPath(srcBucket, srcPath)
|
||||
|
||||
+44
-1
@@ -262,6 +262,9 @@ func (s *S3Storage) Remove(ctx context.Context, bucket, fnm string, tenantID ...
|
||||
Key: aws.String(fnm),
|
||||
})
|
||||
if err != nil {
|
||||
if isS3NotFound(err) {
|
||||
return nil
|
||||
}
|
||||
common.Error("Failed to remove object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
@@ -287,6 +290,18 @@ func (s *S3Storage) ObjExist(ctx context.Context, bucket, fnm string, tenantID .
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *S3Storage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
|
||||
bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
|
||||
_, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(fnm)})
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if isS3NotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
func (s *S3Storage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
|
||||
|
||||
listInput := &s3.ListObjectsV2Input{
|
||||
@@ -349,6 +364,11 @@ func (s *S3Storage) BucketExists(ctx context.Context, bucket string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *S3Storage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
|
||||
actualBucket, _ := s.resolveBucketAndPrefix(bucket)
|
||||
return s.bucketExistsForRemoval(ctx, actualBucket)
|
||||
}
|
||||
|
||||
// RemoveBucket removes a bucket and all its objects
|
||||
func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {
|
||||
actualBucket, prefix := s.resolveBucketAndPrefix(bucket)
|
||||
@@ -378,6 +398,29 @@ func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveEmptyBucket removes a bucket only when it contains no object versions.
|
||||
func (s *S3Storage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
|
||||
actualBucket, prefix := s.resolveBucketAndPrefix(bucket)
|
||||
exists, err := s.bucketExistsForRemoval(ctx, actualBucket)
|
||||
if err != nil || !exists {
|
||||
return err
|
||||
}
|
||||
if s.bucket != "" {
|
||||
versions, err := s.client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(actualBucket), Prefix: aws.String(prefix), MaxKeys: aws.Int32(1),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(versions.Versions) > 0 || len(versions.DeleteMarkers) > 0 || (versions.IsTruncated != nil && *versions.IsTruncated) {
|
||||
return fmt.Errorf("bucket %s is not empty", bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
_, err = s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(actualBucket)})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3Storage) resolveBucketAndPrefix(bucket string) (string, string) {
|
||||
if s.bucket == "" {
|
||||
return bucket, ""
|
||||
@@ -490,7 +533,7 @@ func isS3NotFound(err error) bool {
|
||||
}
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchKey"
|
||||
return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchKey" || apiErr.ErrorCode() == "NoSuchBucket"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -129,6 +129,93 @@ func TestS3StorageRemoveBucketDeletesEmptyPhysicalBucket(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3StorageRemoveEmptyBucketPreservesNonemptyPhysicalBucket(t *testing.T) {
|
||||
storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
|
||||
switch req.Method {
|
||||
case http.MethodHead:
|
||||
return s3Response(http.StatusOK, ""), nil
|
||||
case http.MethodDelete:
|
||||
return s3Response(http.StatusConflict, `<Error><Code>BucketNotEmpty</Code></Error>`), nil
|
||||
}
|
||||
t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
|
||||
return nil, nil
|
||||
})
|
||||
if err := storage.RemoveEmptyBucket(t.Context(), "kb01"); err == nil {
|
||||
t.Fatal("RemoveEmptyBucket returned nil for a nonempty bucket")
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3StorageExistenceChecksDistinguishMissingFromFailure(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
status int
|
||||
code string
|
||||
missing bool
|
||||
}{
|
||||
{name: "missing", status: http.StatusNotFound, code: "NoSuchBucket", missing: true},
|
||||
{name: "unavailable", status: http.StatusInternalServerError, code: "InternalError"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
|
||||
return s3Response(test.status, `<Error><Code>`+test.code+`</Code></Error>`), nil
|
||||
})
|
||||
objectExists, objectErr := storage.ObjectExists(t.Context(), "kb01", "file")
|
||||
bucketExists, bucketErr := storage.BucketExistsWithError(t.Context(), "kb01")
|
||||
if objectExists || bucketExists {
|
||||
t.Fatal("existence check reported an existing item")
|
||||
}
|
||||
if test.missing && (objectErr != nil || bucketErr != nil) {
|
||||
t.Fatalf("missing item returned errors: object=%v bucket=%v", objectErr, bucketErr)
|
||||
}
|
||||
if !test.missing && (objectErr == nil || bucketErr == nil) {
|
||||
t.Fatalf("storage failure was treated as missing: object=%v bucket=%v", objectErr, bucketErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3StorageRemoveEmptyBucketDeletesEmptyPhysicalBucket(t *testing.T) {
|
||||
deleted := false
|
||||
storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
|
||||
switch req.Method {
|
||||
case http.MethodHead:
|
||||
return s3Response(http.StatusOK, ""), nil
|
||||
case http.MethodDelete:
|
||||
deleted = true
|
||||
return s3Response(http.StatusNoContent, ""), nil
|
||||
}
|
||||
t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
|
||||
return nil, nil
|
||||
})
|
||||
if err := storage.RemoveEmptyBucket(t.Context(), "kb01"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !deleted {
|
||||
t.Fatal("RemoveEmptyBucket did not delete an empty bucket")
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3StorageRemoveEmptyBucketPreservesLogicalBucketObjects(t *testing.T) {
|
||||
storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case req.Method == http.MethodHead:
|
||||
return s3Response(http.StatusOK, ""), nil
|
||||
case req.URL.Query().Has("versions"):
|
||||
if got := req.URL.Query().Get("prefix"); got != "prefix/kb01/" {
|
||||
t.Fatalf("prefix = %q", got)
|
||||
}
|
||||
return s3Response(http.StatusOK, `<ListVersionsResult><Version><Key>prefix/kb01/file</Key><VersionId>v1</VersionId></Version></ListVersionsResult>`), nil
|
||||
}
|
||||
t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
|
||||
return nil, nil
|
||||
})
|
||||
storage.bucket = "physical"
|
||||
storage.prefixPath = "prefix"
|
||||
if err := storage.RemoveEmptyBucket(t.Context(), "kb01"); err == nil {
|
||||
t.Fatal("RemoveEmptyBucket returned nil for a nonempty logical bucket")
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3StorageRemoveBucketSingleBucketMode(t *testing.T) {
|
||||
var deleteBody string
|
||||
storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
|
||||
|
||||
@@ -78,6 +78,8 @@ type Storage interface {
|
||||
|
||||
// ObjExist checks if an object exists
|
||||
ObjExist(ctx context.Context, bucket, fnm string, tenantID ...string) bool
|
||||
// ObjectExists distinguishes a missing object from a storage error
|
||||
ObjectExists(ctx context.Context, bucket, fnm string) (bool, error)
|
||||
|
||||
// ListObjects list all objects of the bucket
|
||||
ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error)
|
||||
@@ -88,10 +90,15 @@ type Storage interface {
|
||||
|
||||
// BucketExists checks if a bucket exists
|
||||
BucketExists(ctx context.Context, bucket string) bool
|
||||
// BucketExistsWithError distinguishes a missing bucket from a storage error
|
||||
BucketExistsWithError(ctx context.Context, bucket string) (bool, error)
|
||||
|
||||
// RemoveBucket removes a bucket and all its objects
|
||||
RemoveBucket(ctx context.Context, bucket string) error
|
||||
|
||||
// RemoveEmptyBucket removes a bucket only if it has no objects
|
||||
RemoveEmptyBucket(ctx context.Context, bucket string) error
|
||||
|
||||
// Copy copies an object from source to destination
|
||||
Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool
|
||||
|
||||
|
||||
Reference in New Issue
Block a user