Feature: delete user all stuffs when delete user (#20010)

This commit is contained in:
Wang Qi
2026-09-23 09:31:39 +08:00
committed by GitHub
parent 3a23ba00a5
commit 769d091d7a
24 changed files with 1891 additions and 252 deletions
+7 -190
View File
@@ -74,6 +74,8 @@ type Service struct {
BillingSubscriptionDAO *dao.BillingSubscriptionDAO
MemoryDAO *dao.MemoryDAO
SearchDAO *dao.SearchDAO
deleteEngine engine.DocEngine
deleteStorage storage.Storage
}
// NewService create admin service
@@ -524,12 +526,11 @@ type DeleteUserResult struct {
// - *DeleteUserResult
// - error: error message
func (s *Service) DeleteUser(ctx context.Context, username string) (*DeleteUserResult, error) {
result := &DeleteUserResult{
Username: username,
DeletedDetails: []string{fmt.Sprintf("Drop user: %s", username)},
}
userList, err := s.userDAO.ListByEmail(ctx, dao.DB, username)
if err != nil || len(userList) == 0 {
if err != nil {
return nil, fmt.Errorf("failed to get user: %w", err)
}
if len(userList) == 0 {
return nil, fmt.Errorf("user '%s' not found", username)
}
@@ -549,191 +550,7 @@ func (s *Service) DeleteUser(ctx context.Context, username string) (*DeleteUserR
return nil, fmt.Errorf("user '%s' is admin account and cannot be deleted", username)
}
// Get user-tenant relations
tenants, err := s.userTenantDAO.GetByUserIDAll(ctx, dao.DB, user.ID)
if err != nil {
common.Warn("failed to get user-tenant relations", zap.Error(err))
}
// Find owned tenant (role = "owner")
var ownedTenantID string
for _, t := range tenants {
if t.Role == "owner" {
ownedTenantID = t.TenantID
break
}
}
// Start transaction for cascade delete
tx := dao.DB.Begin()
if tx.Error != nil {
return nil, fmt.Errorf("failed to begin transaction: %w", tx.Error)
}
// Rollback helper function
rollbackTx := func() {
if rbErr := tx.Rollback(); rbErr.Error != nil {
common.Error("failed to rollback transaction", rbErr.Error)
}
}
result.DeletedDetails = append(result.DeletedDetails, "Start to delete owned tenant.")
// Delete owned tenant data
if ownedTenantID != "" {
// 1. Get knowledge base IDs
kbIDs, err := s.kbDAO.GetKBIDsByTenantIDSimple(ctx, tx, ownedTenantID)
if err != nil {
common.Warn("failed to get knowledge base IDs", zap.Error(err))
}
if len(kbIDs) > 0 {
// 2. Get document IDs
docIDs, err := s.documentDAO.GetAllDocIDsByKBIDs(ctx, tx, kbIDs)
if err != nil {
common.Warn("failed to get document IDs", zap.Error(err))
}
// 3. Delete tasks by document IDs
if len(docIDs) > 0 {
docIDList := make([]string, len(docIDs))
for i, d := range docIDs {
docIDList[i] = d["id"]
}
if delErr := tx.Unscoped().Where("doc_id IN ?", docIDList).Delete(&entity.Task{}); delErr.Error != nil {
common.Warn("failed to delete tasks", zap.Error(delErr.Error))
}
}
// 4. Delete documents
if delErr := tx.Unscoped().Where("kb_id IN ?", kbIDs).Delete(&entity.Document{}); delErr.Error != nil {
common.Warn("failed to delete documents", zap.Error(delErr.Error))
}
// 5. Delete knowledge bases
if delErr := tx.Unscoped().Where("id IN ?", kbIDs).Delete(&entity.Knowledgebase{}); delErr.Error != nil {
common.Warn("failed to delete knowledge bases", zap.Error(delErr.Error))
}
}
// 6. Delete files
if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.File{}); delErr.Error != nil {
common.Warn("failed to delete files", zap.Error(delErr.Error))
}
// 7. Delete user canvas (agents)
if delErr := tx.Unscoped().Where("user_id = ?", ownedTenantID).Delete(&entity.UserCanvas{}); delErr.Error != nil {
common.Warn("failed to delete user canvas", zap.Error(delErr.Error))
}
// 8. Get dialog IDs
var dialogIDs []string
if pluckErr := tx.Model(&entity.Chat{}).Where("tenant_id = ?", ownedTenantID).Pluck("id", &dialogIDs); pluckErr.Error != nil {
common.Warn("failed to get dialog IDs", zap.Error(pluckErr.Error))
}
// 9. Delete chat sessions
if len(dialogIDs) > 0 {
var sessionIDs []string
if pluckErr := tx.Model(&entity.ChatSession{}).Where("dialog_id IN ?", dialogIDs).Pluck("id", &sessionIDs); pluckErr.Error != nil {
common.Warn("failed to get chat session IDs", zap.Error(pluckErr.Error))
}
if len(sessionIDs) > 0 {
if delErr := tx.Table("conversation_message").Where("conversation_id IN ?", sessionIDs).Delete(map[string]interface{}{}); delErr.Error != nil {
common.Warn("failed to delete conversation messages", zap.Error(delErr.Error))
}
if delErr := tx.Table("conversation_reference").Where("conversation_id IN ?", sessionIDs).Delete(map[string]interface{}{}); delErr.Error != nil {
common.Warn("failed to delete conversation references", zap.Error(delErr.Error))
}
}
if delErr := tx.Unscoped().Where("dialog_id IN ?", dialogIDs).Delete(&entity.ChatSession{}); delErr.Error != nil {
common.Warn("failed to delete chat sessions", zap.Error(delErr.Error))
}
}
// 10. Delete chats/dialogs
if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.Chat{}); delErr.Error != nil {
common.Warn("failed to delete chats", zap.Error(delErr.Error))
}
// 11. Delete API tokens
if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.APIToken{}); delErr.Error != nil {
common.Warn("failed to delete API tokens", zap.Error(delErr.Error))
}
// 12. Delete API4Conversations
if len(dialogIDs) > 0 {
var sessionIDs []string
if pluckErr := tx.Model(&entity.API4Conversation{}).Where("dialog_id IN ?", dialogIDs).Pluck("id", &sessionIDs); pluckErr.Error != nil {
common.Warn("failed to get API conversation IDs", zap.Error(pluckErr.Error))
}
if len(sessionIDs) > 0 {
if delErr := tx.Table("api_4_conversation_message").Where("conversation_id IN ?", sessionIDs).Delete(map[string]interface{}{}); delErr.Error != nil {
common.Warn("failed to delete API conversation messages", zap.Error(delErr.Error))
}
if delErr := tx.Table("api_4_conversation_reference").Where("conversation_id IN ?", sessionIDs).Delete(map[string]interface{}{}); delErr.Error != nil {
common.Warn("failed to delete API conversation references", zap.Error(delErr.Error))
}
}
if delErr := tx.Unscoped().Where("dialog_id IN ?", dialogIDs).Delete(&entity.API4Conversation{}); delErr.Error != nil {
common.Warn("failed to delete API4Conversations", zap.Error(delErr.Error))
}
}
var tenantLLMCount int64
tx.Model(&entity.TenantLLM{}).Where("tenant_id = ?", ownedTenantID).Count(&tenantLLMCount)
result.TenantLLMCount = int(tenantLLMCount)
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d tenant-LLM records.", tenantLLMCount))
result.LangfuseCount = 0
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d langfuse records.", result.LangfuseCount))
metadataTableName := fmt.Sprintf("ragflow_doc_meta_%s", ownedTenantID[:32])
result.MetadataTable = metadataTableName
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted metadata table %s.", metadataTableName))
// 13. Delete tenant LLM configurations
if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.TenantLLM{}); delErr.Error != nil {
common.Warn("failed to delete tenant LLM", zap.Error(delErr.Error))
}
var tenantCount int64
tx.Model(&entity.Tenant{}).Where("id = ?", ownedTenantID).Count(&tenantCount)
result.TenantCount = int(tenantCount)
// 14. Delete tenant
if delErr := tx.Unscoped().Where("id = ?", ownedTenantID).Delete(&entity.Tenant{}); delErr.Error != nil {
common.Warn("failed to delete tenant", zap.Error(delErr.Error))
}
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d tenant.", result.TenantCount))
}
var userTenantCount int64
tx.Model(&entity.UserTenant{}).Where("user_id = ?", user.ID).Count(&userTenantCount)
result.UserTenantCount = int(userTenantCount)
// 15. Delete user-tenant relations
if delErr := tx.Unscoped().Where("user_id = ?", user.ID).Delete(&entity.UserTenant{}); delErr.Error != nil {
common.Warn("failed to delete user-tenant relations", zap.Error(delErr.Error))
}
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d user-tenant records.", result.UserTenantCount))
result.UserCount = 1
// 16. Finally, hard delete user
if delErr := tx.Unscoped().Where("id = ?", user.ID).Delete(&entity.User{}); delErr.Error != nil {
rollbackTx()
return nil, fmt.Errorf("failed to delete user: %w", delErr.Error)
}
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d user.", result.UserCount))
// Commit transaction
if commitErr := tx.Commit(); commitErr.Error != nil {
return nil, fmt.Errorf("failed to commit transaction: %w", commitErr.Error)
}
result.DeletedDetails = append(result.DeletedDetails, "Delete done!")
common.Info("Delete user success with all related data", zap.String("username", username))
return result, nil
return s.deleteUserData(ctx, user)
}
// ChangePassword change user password
+760
View File
@@ -0,0 +1,760 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package admin
import (
"context"
"fmt"
"time"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/engine"
"ragflow/internal/entity"
"ragflow/internal/ingestion/knowledge_compile"
servicepkg "ragflow/internal/service"
"ragflow/internal/storage"
"go.uber.org/zap"
"gorm.io/gorm"
)
type userDeletionData struct {
ownedTenantID string
datasets []entity.Knowledgebase
documents []entity.Document
files []entity.File
memories []entity.Memory
spaces []entity.SkillSpace
datasetIDs []string
documentIDs []string
fileIDs []string
docTenants map[string]string
datasetNames map[string]string
}
func namedID(name, id string) string {
if name == "" {
return id
}
return fmt.Sprintf("%s (%s)", name, id)
}
func namedDocument(document entity.Document) string {
if document.Name == nil {
return document.ID
}
return namedID(*document.Name, document.ID)
}
func (s *Service) deleteUserData(ctx context.Context, user *entity.User) (*DeleteUserResult, error) {
data, err := loadUserDeletionData(ctx, user.ID)
if err != nil {
return nil, err
}
docEngine := s.deleteEngine
if docEngine == nil {
docEngine = engine.Get()
}
store := s.deleteStorage
if store == nil {
store = storage.GetStorageFactory().GetStorage()
}
if err := data.deleteExternalData(ctx, docEngine, store); err != nil {
return nil, err
}
result := &DeleteUserResult{
Username: user.Email,
DeletedDetails: []string{fmt.Sprintf("Drop user: %s", user.Email)},
}
if err := dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
return data.deleteDatabaseRows(ctx, tx, user.ID, result)
}); err != nil {
return nil, fmt.Errorf("failed to delete user data: %w", err)
}
datasetIDs := make(map[string]struct{}, len(data.datasetIDs))
for _, id := range data.datasetIDs {
datasetIDs[id] = struct{}{}
}
publishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
defer cancel()
publishWarning := false
for _, document := range data.documents {
if _, deletingDataset := datasetIDs[document.KbID]; deletingDataset {
continue
}
if err := publishCtx.Err(); err != nil {
common.Warn("stopped publishing document deletions", zap.Error(err))
publishWarning = true
break
}
if err := knowledge_compile.PublishDeleted(publishCtx, data.docTenants[document.KbID], document.KbID, document.ID, nil, nil); err != nil {
common.Warn("failed to publish document deletion", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.Error(err))
publishWarning = true
}
}
if publishWarning {
result.DeletedDetails = append(result.DeletedDetails, "- Warning: dataset artifact refresh could not be scheduled for some deleted documents.")
}
result.DeletedDetails = append(result.DeletedDetails, "Delete done!")
common.Info("Delete user success with all related data")
return result, nil
}
func loadUserDeletionData(ctx context.Context, userID string) (*userDeletionData, error) {
db := dao.DB.WithContext(ctx)
var tenants []entity.UserTenant
if err := db.Where("user_id = ? AND role = ?", userID, "owner").Find(&tenants).Error; err != nil {
return nil, fmt.Errorf("load owned tenant: %w", err)
}
if len(tenants) > 1 {
return nil, fmt.Errorf("user %s owns more than one tenant", userID)
}
data := &userDeletionData{}
if len(tenants) == 1 {
data.ownedTenantID = tenants[0].TenantID
}
datasetQuery := db.Model(&entity.Knowledgebase{}).Select("id", "tenant_id", "created_by", "name").Where("created_by = ?", userID)
if data.ownedTenantID != "" {
datasetQuery = datasetQuery.Or("tenant_id = ?", data.ownedTenantID)
}
if err := datasetQuery.Find(&data.datasets).Error; err != nil {
return nil, fmt.Errorf("load datasets: %w", err)
}
data.datasetNames = make(map[string]string, len(data.datasets))
for _, dataset := range data.datasets {
data.datasetIDs = append(data.datasetIDs, dataset.ID)
data.datasetNames[dataset.ID] = dataset.Name
}
if err := db.Model(&entity.Document{}).Select("id", "kb_id", "name", "location", "token_num", "chunk_num").Where("created_by = ?", userID).Find(&data.documents).Error; err != nil {
return nil, fmt.Errorf("load documents: %w", err)
}
seenDocuments := make(map[string]struct{}, len(data.documents))
for _, document := range data.documents {
seenDocuments[document.ID] = struct{}{}
}
for start := 0; start < len(data.datasetIDs); start += 1000 {
end := min(start+1000, len(data.datasetIDs))
var documents []entity.Document
if err := db.Model(&entity.Document{}).Select("id", "kb_id", "name", "location", "token_num", "chunk_num").Where("kb_id IN ?", data.datasetIDs[start:end]).Find(&documents).Error; err != nil {
return nil, fmt.Errorf("load dataset documents: %w", err)
}
for _, document := range documents {
if _, seen := seenDocuments[document.ID]; !seen {
data.documents = append(data.documents, document)
seenDocuments[document.ID] = struct{}{}
}
}
}
for _, document := range data.documents {
data.documentIDs = append(data.documentIDs, document.ID)
}
data.docTenants = make(map[string]string)
if len(data.documents) > 0 {
deletingDatasets := make(map[string]struct{}, len(data.datasetIDs))
for _, id := range data.datasetIDs {
deletingDatasets[id] = struct{}{}
}
remainingDatasetIDs := make(map[string]struct{})
for _, document := range data.documents {
if _, deleting := deletingDatasets[document.KbID]; !deleting {
remainingDatasetIDs[document.KbID] = struct{}{}
}
}
if len(remainingDatasetIDs) > 0 {
ids := make([]string, 0, len(remainingDatasetIDs))
for id := range remainingDatasetIDs {
ids = append(ids, id)
}
for start := 0; start < len(ids); start += 1000 {
end := min(start+1000, len(ids))
var datasets []entity.Knowledgebase
if err := db.Select("id", "tenant_id", "name").Where("id IN ?", ids[start:end]).Find(&datasets).Error; err != nil {
return nil, fmt.Errorf("load document tenants: %w", err)
}
for _, dataset := range datasets {
data.docTenants[dataset.ID] = dataset.TenantID
data.datasetNames[dataset.ID] = dataset.Name
}
}
if len(data.docTenants) != len(ids) {
return nil, fmt.Errorf("some documents refer to missing datasets")
}
}
}
fileQuery := db.Model(&entity.File{}).Select("id", "parent_id", "tenant_id", "name", "location", "type", "source_type").Where("created_by = ?", userID)
if data.ownedTenantID != "" {
fileQuery = fileQuery.Or("tenant_id = ?", data.ownedTenantID)
}
if err := fileQuery.Find(&data.files).Error; err != nil {
return nil, fmt.Errorf("load files: %w", err)
}
for _, file := range data.files {
data.fileIDs = append(data.fileIDs, file.ID)
}
if data.ownedTenantID != "" {
if err := db.Select("id", "tenant_id").Where("tenant_id = ?", data.ownedTenantID).Find(&data.memories).Error; err != nil {
return nil, fmt.Errorf("load memories: %w", err)
}
if err := db.Select("id", "tenant_id").Where("tenant_id = ?", data.ownedTenantID).Find(&data.spaces).Error; err != nil {
return nil, fmt.Errorf("load skill spaces: %w", err)
}
}
return data, nil
}
func (data *userDeletionData) deleteExternalData(ctx context.Context, docEngine engine.DocEngine, store storage.Storage) error {
// External stores cannot join the SQL transaction. Keep the user row until
// cleanup succeeds so a failed request can be retried.
if docEngine == nil && (data.ownedTenantID != "" || len(data.datasets) > 0 || len(data.documents) > 0) {
return fmt.Errorf("document engine is unavailable for user data cleanup")
}
if store == nil && (len(data.datasets) > 0 || len(data.documents) > 0 || len(data.files) > 0) {
return fmt.Errorf("storage is unavailable for user data cleanup")
}
datasetIDs := make(map[string]struct{}, len(data.datasetIDs))
for _, id := range data.datasetIDs {
datasetIDs[id] = struct{}{}
}
for _, document := range data.documents {
if document.Location == nil || *document.Location == "" {
continue
}
exists, err := store.ObjectExists(ctx, document.KbID, *document.Location)
if err != nil {
return fmt.Errorf("check document %s: %w", document.ID, err)
}
if !exists {
common.Warn("Document object already missing", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))
continue
}
if err := store.Remove(ctx, document.KbID, *document.Location); err != nil {
return fmt.Errorf("remove document %s: %w", document.ID, err)
}
common.Info("Removed document object", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))
}
for _, dataset := range data.datasets {
exists, err := store.BucketExistsWithError(ctx, dataset.ID)
if err != nil {
return fmt.Errorf("check dataset bucket %s: %w", dataset.ID, err)
}
if !exists {
common.Warn("Dataset bucket already missing", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID))
continue
}
if err := store.RemoveEmptyBucket(ctx, dataset.ID); err != nil {
common.Warn("Unable to remove empty dataset bucket", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID), zap.Error(err))
} else {
common.Info("Removed empty dataset bucket", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID))
}
}
for _, file := range data.files {
if file.SourceType != string(entity.FileSourceKnowledgebase) && file.Location != nil && *file.Location != "" && file.Type != "folder" {
exists, err := store.ObjectExists(ctx, file.ParentID, *file.Location)
if err != nil {
return fmt.Errorf("check file %s: %w", file.ID, err)
}
if !exists {
common.Warn("File object already missing", zap.String("file", namedID(file.Name, file.ID)), zap.String("file_id", file.ID), zap.String("bucket", file.ParentID))
continue
}
if err := store.Remove(ctx, file.ParentID, *file.Location); err != nil {
return fmt.Errorf("remove file %s: %w", file.ID, err)
}
common.Info("Removed file object", zap.String("file", namedID(file.Name, file.ID)), zap.String("file_id", file.ID), zap.String("bucket", file.ParentID))
}
}
for _, file := range data.files {
if file.Type != "folder" || file.SourceType == string(entity.FileSourceKnowledgebase) || file.TenantID != data.ownedTenantID {
continue
}
exists, err := store.BucketExistsWithError(ctx, file.ID)
if err != nil {
return fmt.Errorf("check folder bucket %s: %w", file.ID, err)
}
if !exists {
common.Warn("Folder bucket already missing", zap.String("folder", namedID(file.Name, file.ID)), zap.String("bucket", file.ID))
continue
}
if err := store.RemoveEmptyBucket(ctx, file.ID); err != nil {
common.Warn("Failed to remove empty folder bucket", zap.String("folder", namedID(file.Name, file.ID)), zap.String("bucket", file.ID), zap.Error(err))
} else {
common.Info("Removed empty folder bucket", zap.String("folder", namedID(file.Name, file.ID)), zap.String("bucket", file.ID))
}
}
if docEngine == nil {
return nil
}
if data.ownedTenantID != "" {
ownedDatasetIDs := make([]string, 0)
for _, dataset := range data.datasets {
if dataset.TenantID == data.ownedTenantID {
ownedDatasetIDs = append(ownedDatasetIDs, dataset.ID)
}
}
if err := dropUserChunkStore(ctx, docEngine, servicepkg.IndexName(data.ownedTenantID), ownedDatasetIDs); err != nil {
return fmt.Errorf("remove tenant chunks: %w", err)
}
if err := dropUserChunkStore(ctx, docEngine, servicepkg.MemoryIndexName(data.ownedTenantID), memoryIDs(data.memories)); err != nil {
return fmt.Errorf("remove memory messages: %w", err)
}
if err := docEngine.DropMetadataStore(ctx, data.ownedTenantID); err != nil {
return fmt.Errorf("remove tenant metadata: %w", err)
}
}
for _, dataset := range data.datasets {
if dataset.TenantID != data.ownedTenantID {
indexName := servicepkg.IndexName(dataset.TenantID)
exists, err := docEngine.ChunkStoreExists(ctx, indexName, dataset.ID)
if err != nil {
return fmt.Errorf("check dataset chunks %s: %w", dataset.ID, err)
}
if !exists {
continue
}
if _, err := docEngine.DeleteChunks(ctx, map[string]interface{}{"kb_id": dataset.ID}, indexName, dataset.ID); err != nil {
return fmt.Errorf("remove dataset chunks %s: %w", dataset.ID, err)
}
}
}
documentsByDataset := make(map[string][]string)
for _, document := range data.documents {
if _, deletingDataset := datasetIDs[document.KbID]; !deletingDataset {
documentsByDataset[document.KbID] = append(documentsByDataset[document.KbID], document.ID)
}
}
for kbID, docIDs := range documentsByDataset {
tenantID := data.docTenants[kbID]
if tenantID == "" {
return fmt.Errorf("dataset %s for documents not found", kbID)
}
indexName := servicepkg.IndexName(tenantID)
exists, err := docEngine.ChunkStoreExists(ctx, indexName, kbID)
if err != nil {
return fmt.Errorf("check document chunks for dataset %s: %w", kbID, err)
}
if exists {
for start := 0; start < len(docIDs); start += 1000 {
end := min(start+1000, len(docIDs))
if _, err := docEngine.DeleteChunks(ctx, map[string]interface{}{"doc_id": docIDs[start:end]}, indexName, kbID); err != nil {
return fmt.Errorf("remove document chunks for dataset %s: %w", kbID, err)
}
}
}
metadataExists, err := docEngine.MetadataStoreExists(ctx, tenantID)
if err != nil {
return fmt.Errorf("check document metadata for dataset %s: %w", kbID, err)
}
if metadataExists {
for start := 0; start < len(docIDs); start += 1000 {
end := min(start+1000, len(docIDs))
if _, err := docEngine.DeleteMetadata(ctx, map[string]interface{}{"id": docIDs[start:end]}, tenantID); err != nil {
return fmt.Errorf("remove document metadata for dataset %s: %w", kbID, err)
}
}
}
}
for _, space := range data.spaces {
if err := docEngine.DropChunkStore(ctx, servicepkg.SkillIndexName(space.TenantID, space.ID), "skill"); err != nil {
return fmt.Errorf("remove skill index %s: %w", space.ID, err)
}
}
return nil
}
func dropUserChunkStore(ctx context.Context, docEngine engine.DocEngine, indexName string, ids []string) error {
if docEngine.GetType() == string(engine.EngineInfinity) {
for _, id := range ids {
if err := docEngine.DropChunkStore(ctx, indexName, id); err != nil {
return err
}
}
return nil
}
return docEngine.DropChunkStore(ctx, indexName, "")
}
func memoryIDs(memories []entity.Memory) []string {
ids := make([]string, 0, len(memories))
for _, memory := range memories {
ids = append(ids, memory.ID)
}
return ids
}
func (data *userDeletionData) deleteDatabaseRows(ctx context.Context, tx *gorm.DB, userID string, result *DeleteUserResult) error {
selectIDs := func(model interface{}, condition string, args ...interface{}) ([]string, error) {
var ids []string
err := tx.WithContext(ctx).Model(model).Where(condition, args...).Pluck("id", &ids).Error
return ids, err
}
remove := func(label string, model interface{}, condition string, args ...interface{}) (int64, error) {
deleted := tx.WithContext(ctx).Unscoped().Where(condition, args...).Delete(model)
if deleted.Error != nil {
return 0, fmt.Errorf("delete %s: %w", label, deleted.Error)
}
if deleted.RowsAffected > 0 {
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d %s records.", deleted.RowsAffected, label))
}
return deleted.RowsAffected, nil
}
removeIDs := func(label string, model interface{}, column string, ids []string) error {
var total int64
for start := 0; start < len(ids); start += 1000 {
end := min(start+1000, len(ids))
deleted := tx.WithContext(ctx).Unscoped().Where(column+" IN ?", ids[start:end]).Delete(model)
if deleted.Error != nil {
return fmt.Errorf("delete %s: %w", label, deleted.Error)
}
total += deleted.RowsAffected
}
if total > 0 {
result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d %s records.", total, label))
}
return nil
}
owner := data.ownedTenantID
var chatIDs, canvasIDs, conversationIDs, apiConversationIDs []string
var evaluationDatasetIDs, evaluationRunIDs, connectorIDs, providerIDs, instanceIDs, modelIDs, groupIDs []string
var ingestionTaskIDs, memoryTaskIDs, commitIDs []string
var err error
if owner != "" {
if chatIDs, err = selectIDs(&entity.Chat{}, "tenant_id = ?", owner); err != nil {
return err
}
if evaluationDatasetIDs, err = selectIDs(&entity.EvaluationDataset{}, "tenant_id = ? OR created_by = ?", owner, userID); err != nil {
return err
}
if connectorIDs, err = selectIDs(&entity.Connector{}, "tenant_id = ?", owner); err != nil {
return err
}
if providerIDs, err = selectIDs(&entity.TenantModelProvider{}, "tenant_id = ?", owner); err != nil {
return err
}
} else if evaluationDatasetIDs, err = selectIDs(&entity.EvaluationDataset{}, "created_by = ?", userID); err != nil {
return err
}
if canvasIDs, err = selectIDs(&entity.UserCanvas{}, "user_id = ?", userID); err != nil {
return err
}
if len(chatIDs) > 0 {
conversationIDs, err = selectIDs(&entity.ChatSession{}, "dialog_id IN ? OR user_id = ?", chatIDs, userID)
if err != nil {
return err
}
} else {
conversationIDs, err = selectIDs(&entity.ChatSession{}, "user_id = ?", userID)
if err != nil {
return err
}
}
apiDialogIDs := append(append([]string{}, chatIDs...), canvasIDs...)
if len(apiDialogIDs) > 0 {
apiConversationIDs, err = selectIDs(&entity.API4Conversation{}, "dialog_id IN ? OR user_id = ?", apiDialogIDs, userID)
} else {
apiConversationIDs, err = selectIDs(&entity.API4Conversation{}, "user_id = ?", userID)
}
if err != nil {
return err
}
if len(evaluationDatasetIDs) > 0 {
evaluationRunIDs, err = selectIDs(&entity.EvaluationRun{}, "dataset_id IN ? OR created_by = ?", evaluationDatasetIDs, userID)
} else {
evaluationRunIDs, err = selectIDs(&entity.EvaluationRun{}, "created_by = ?", userID)
}
if err != nil {
return err
}
if len(providerIDs) > 0 {
if instanceIDs, err = selectIDs(&entity.TenantModelInstance{}, "provider_id IN ?", providerIDs); err != nil {
return err
}
if modelIDs, err = selectIDs(&entity.TenantModel{}, "provider_id IN ?", providerIDs); err != nil {
return err
}
if err = tx.Model(&entity.TenantModelGroupMapping{}).Where("provider_id IN ?", providerIDs).Distinct().Pluck("group_id", &groupIDs).Error; err != nil {
return err
}
}
if len(data.documentIDs) > 0 || len(data.datasetIDs) > 0 {
query := tx.Model(&entity.IngestionTask{}).Where("user_id = ?", userID)
if len(data.documentIDs) > 0 {
query = query.Or("document_id IN ?", data.documentIDs)
}
if len(data.datasetIDs) > 0 {
query = query.Or("dataset_id IN ?", data.datasetIDs)
}
err = query.Pluck("id", &ingestionTaskIDs).Error
} else {
ingestionTaskIDs, err = selectIDs(&entity.IngestionTask{}, "user_id = ?", userID)
}
if err != nil {
return err
}
if len(data.memories) > 0 {
if err = tx.Model(&entity.MemoryTask{}).Where("memory_id IN ?", memoryIDs(data.memories)).Pluck("task_id", &memoryTaskIDs).Error; err != nil {
return err
}
}
commitQuery := tx.Model(&entity.FileCommit{}).Where("author_id = ?", userID)
if len(data.fileIDs) > 0 {
commitQuery = commitQuery.Or("folder_id IN ?", data.fileIDs)
}
if err = commitQuery.Pluck("id", &commitIDs).Error; err != nil {
return err
}
for _, child := range []struct {
label string
model interface{}
ids []string
}{
{"conversation messages", &entity.ConversationMessage{}, conversationIDs},
{"conversation references", &entity.ConversationReference{}, conversationIDs},
{"API conversation messages", &entity.API4ConversationMessage{}, apiConversationIDs},
{"API conversation references", &entity.API4ConversationReference{}, apiConversationIDs},
} {
if err := removeIDs(child.label, child.model, "conversation_id", child.ids); err != nil {
return err
}
}
if err := removeIDs("conversations", &entity.ChatSession{}, "id", conversationIDs); err != nil {
return err
}
if err := removeIDs("API conversations", &entity.API4Conversation{}, "id", apiConversationIDs); err != nil {
return err
}
if err := removeIDs("chat channels", &entity.ChatChannel{}, "chat_id", chatIDs); err != nil {
return err
}
if err := removeIDs("chats", &entity.Chat{}, "id", chatIDs); err != nil {
return err
}
if err := removeIDs("agent versions", &entity.UserCanvasVersion{}, "user_canvas_id", canvasIDs); err != nil {
return err
}
if err := removeIDs("agents", &entity.UserCanvas{}, "id", canvasIDs); err != nil {
return err
}
if err := removeIDs("evaluation results", &entity.EvaluationResult{}, "run_id", evaluationRunIDs); err != nil {
return err
}
if err := removeIDs("evaluation runs", &entity.EvaluationRun{}, "id", evaluationRunIDs); err != nil {
return err
}
if len(evaluationDatasetIDs) > 0 {
var caseIDs []string
if caseIDs, err = selectIDs(&entity.EvaluationCase{}, "dataset_id IN ?", evaluationDatasetIDs); err != nil {
return err
}
if err := removeIDs("evaluation results", &entity.EvaluationResult{}, "case_id", caseIDs); err != nil {
return err
}
if err := removeIDs("evaluation cases", &entity.EvaluationCase{}, "id", caseIDs); err != nil {
return err
}
if err := removeIDs("evaluation datasets", &entity.EvaluationDataset{}, "id", evaluationDatasetIDs); err != nil {
return err
}
}
if err := removeIDs("ingestion task logs", &entity.IngestionTaskLog{}, "task_id", ingestionTaskIDs); err != nil {
return err
}
if err := removeIDs("ingestion tasks", &entity.IngestionTask{}, "id", ingestionTaskIDs); err != nil {
return err
}
if err := removeIDs("memory tasks", &entity.MemoryTask{}, "memory_id", memoryIDs(data.memories)); err != nil {
return err
}
if err := removeIDs("memory task records", &entity.Task{}, "id", memoryTaskIDs); err != nil {
return err
}
if err := removeIDs("document tasks", &entity.Task{}, "doc_id", data.documentIDs); err != nil {
return err
}
if err := removeIDs("document file links", &entity.File2Document{}, "document_id", data.documentIDs); err != nil {
return err
}
if err := removeIDs("file document links", &entity.File2Document{}, "file_id", data.fileIDs); err != nil {
return err
}
if err := removeIDs("documents", &entity.Document{}, "id", data.documentIDs); err != nil {
return err
}
if err := removeIDs("file commit items", &entity.FileCommitItem{}, "commit_id", commitIDs); err != nil {
return err
}
if err := removeIDs("file commit items", &entity.FileCommitItem{}, "file_id", data.fileIDs); err != nil {
return err
}
if err := removeIDs("file commits", &entity.FileCommit{}, "id", commitIDs); err != nil {
return err
}
if err := removeIDs("files", &entity.File{}, "id", data.fileIDs); err != nil {
return err
}
if err := removeIDs("connector dataset links", &entity.Connector2Kb{}, "connector_id", connectorIDs); err != nil {
return err
}
if err := removeIDs("connector dataset links", &entity.Connector2Kb{}, "kb_id", data.datasetIDs); err != nil {
return err
}
if err := removeIDs("sync logs", &entity.SyncLogs{}, "connector_id", connectorIDs); err != nil {
return err
}
if err := removeIDs("sync logs", &entity.SyncLogs{}, "kb_id", data.datasetIDs); err != nil {
return err
}
if err := removeIDs("connectors", &entity.Connector{}, "id", connectorIDs); err != nil {
return err
}
if err := removeIDs("knowledge compile records", &entity.KnowledgeCompileDataset{}, "dataset_id", data.datasetIDs); err != nil {
return err
}
if err := removeIDs("wiki document records", &entity.WikiDocumentDirty{}, "document_id", data.documentIDs); err != nil {
return err
}
if err := removeIDs("wiki document records", &entity.WikiDocumentDirty{}, "dataset_id", data.datasetIDs); err != nil {
return err
}
if err := removeIDs("pipeline logs", &entity.PipelineOperationLog{}, "document_id", data.documentIDs); err != nil {
return err
}
if err := removeIDs("pipeline logs", &entity.PipelineOperationLog{}, "kb_id", data.datasetIDs); err != nil {
return err
}
if err := removeIDs("datasets", &entity.Knowledgebase{}, "id", data.datasetIDs); err != nil {
return err
}
datasetIDs := make(map[string]struct{}, len(data.datasetIDs))
for _, id := range data.datasetIDs {
datasetIDs[id] = struct{}{}
}
counts := make(map[string]struct{ documents, chunks, tokens int64 })
for _, document := range data.documents {
if _, deletingDataset := datasetIDs[document.KbID]; deletingDataset {
continue
}
count := counts[document.KbID]
count.documents++
count.chunks += document.ChunkNum
count.tokens += document.TokenNum
counts[document.KbID] = count
}
for kbID, count := range counts {
if err := tx.Model(&entity.Knowledgebase{}).Where("id = ?", kbID).Updates(map[string]interface{}{
"doc_num": gorm.Expr("CASE WHEN doc_num >= ? THEN doc_num - ? ELSE 0 END", count.documents, count.documents),
"chunk_num": gorm.Expr("CASE WHEN chunk_num >= ? THEN chunk_num - ? ELSE 0 END", count.chunks, count.chunks),
"token_num": gorm.Expr("CASE WHEN token_num >= ? THEN token_num - ? ELSE 0 END", count.tokens, count.tokens),
}).Error; err != nil {
return fmt.Errorf("update dataset counters: %w", err)
}
}
if len(groupIDs) > 0 {
if err := removeIDs("model group mappings", &entity.TenantModelGroupMapping{}, "provider_id", providerIDs); err != nil {
return err
}
for _, groupID := range groupIDs {
var remaining int64
if err := tx.Model(&entity.TenantModelGroupMapping{}).Where("group_id = ?", groupID).Count(&remaining).Error; err != nil {
return err
}
if remaining == 0 {
if _, err := remove("model groups", &entity.TenantModelGroup{}, "id = ?", groupID); err != nil {
return err
}
}
}
}
if err := removeIDs("models", &entity.TenantModel{}, "id", modelIDs); err != nil {
return err
}
if err := removeIDs("model instances", &entity.TenantModelInstance{}, "id", instanceIDs); err != nil {
return err
}
if err := removeIDs("model providers", &entity.TenantModelProvider{}, "id", providerIDs); err != nil {
return err
}
if owner != "" {
for _, item := range []struct {
label string
model interface{}
}{
{"chat channels", &entity.ChatChannel{}},
{"searches", &entity.Search{}},
{"memories", &entity.Memory{}},
{"MCP servers", &entity.MCPServer{}},
{"skill search configurations", &entity.SkillSearchConfig{}},
{"skill spaces", &entity.SkillSpace{}},
{"compilation templates", &entity.CompilationTemplate{}},
{"compilation template groups", &entity.CompilationTemplateGroup{}},
{"pipeline logs", &entity.PipelineOperationLog{}},
{"knowledge compile records", &entity.KnowledgeCompileDataset{}},
{"wiki document records", &entity.WikiDocumentDirty{}},
{"API tokens", &entity.APIToken{}},
} {
if _, err := remove(item.label, item.model, "tenant_id = ?", owner); err != nil {
return err
}
}
tenantLLMCount, err := remove("tenant LLMs", &entity.TenantLLM{}, "tenant_id = ?", owner)
if err != nil {
return err
}
result.TenantLLMCount = int(tenantLLMCount)
langfuseCount, err := remove("Langfuse credentials", &entity.TenantLangfuse{}, "tenant_id = ?", owner)
if err != nil {
return err
}
result.LangfuseCount = int(langfuseCount)
result.MetadataTable = servicepkg.BuildMetadataIndexName(owner)
ownerMemberships, err := remove("tenant memberships", &entity.UserTenant{}, "tenant_id = ?", owner)
if err != nil {
return err
}
result.UserTenantCount += int(ownerMemberships)
count, err := remove("tenants", &entity.Tenant{}, "id = ?", owner)
if err != nil {
return err
}
result.TenantCount = int(count)
}
if _, err := remove("searches", &entity.Search{}, "created_by = ?", userID); err != nil {
return err
}
invitationCondition := "user_id = ?"
invitationArgs := []interface{}{userID}
if owner != "" {
invitationCondition += " OR tenant_id = ?"
invitationArgs = append(invitationArgs, owner)
}
if _, err := remove("invitations", &entity.InvitationCode{}, invitationCondition, invitationArgs...); err != nil {
return err
}
count, err := remove("user tenant memberships", &entity.UserTenant{}, "user_id = ?", userID)
if err != nil {
return err
}
result.UserTenantCount += int(count)
count, err = remove("users", &entity.User{}, "id = ?", userID)
if err != nil {
return err
}
result.UserCount = int(count)
return nil
}
+384
View File
@@ -0,0 +1,384 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package admin
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"testing"
"ragflow/internal/common"
"ragflow/internal/engine"
"ragflow/internal/entity"
"ragflow/internal/ingestion/testutil"
"ragflow/internal/storage"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"go.uber.org/zap/zaptest/observer"
"gorm.io/gorm"
)
type deletionEngine struct {
engine.DocEngine
dropped []string
deleted []string
}
func (e *deletionEngine) GetType() string { return string(engine.EngineElasticsearch) }
func (e *deletionEngine) DropChunkStore(_ context.Context, name, _ string) error {
e.dropped = append(e.dropped, name)
return nil
}
func (e *deletionEngine) DropMetadataStore(_ context.Context, tenantID string) error {
e.dropped = append(e.dropped, "metadata:"+tenantID)
return nil
}
func (e *deletionEngine) ChunkStoreExists(context.Context, string, string) (bool, error) {
return true, nil
}
func (e *deletionEngine) MetadataStoreExists(context.Context, string) (bool, error) {
return true, nil
}
func (e *deletionEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, _, _ string) (int64, error) {
for key, value := range condition {
e.deleted = append(e.deleted, fmt.Sprintf("%s:%v", key, value))
}
return 1, nil
}
func (e *deletionEngine) DeleteMetadata(_ context.Context, condition map[string]interface{}, _ string) (int64, error) {
for key, value := range condition {
e.deleted = append(e.deleted, fmt.Sprintf("metadata:%s:%v", key, value))
}
return 1, nil
}
type deletionStorage struct {
storage.Storage
buckets []string
files []string
err error
bucketErr error
missing bool
checkErr error
}
func (s *deletionStorage) RemoveBucket(_ context.Context, bucket string) error {
if s.err != nil {
return s.err
}
s.buckets = append(s.buckets, bucket)
return nil
}
func (s *deletionStorage) RemoveEmptyBucket(_ context.Context, bucket string) error {
if s.bucketErr != nil {
return s.bucketErr
}
s.buckets = append(s.buckets, bucket)
return nil
}
func (s *deletionStorage) Remove(_ context.Context, bucket, name string, _ ...string) error {
if s.err != nil {
return s.err
}
s.files = append(s.files, bucket+"/"+name)
return nil
}
func (s *deletionStorage) ObjectExists(_ context.Context, bucket, name string) (bool, error) {
if s.checkErr != nil {
return false, s.checkErr
}
return !s.missing, nil
}
func (s *deletionStorage) BucketExistsWithError(_ context.Context, bucket string) (bool, error) {
if s.checkErr != nil {
return false, s.checkErr
}
return !s.missing, nil
}
func setupUserDeletionDB(t *testing.T) *gorm.DB {
t.Helper()
db := testutil.SetupTestDB(t,
&entity.User{}, &entity.Tenant{}, &entity.UserTenant{},
&entity.Knowledgebase{}, &entity.Document{}, &entity.Task{}, &entity.File{}, &entity.File2Document{},
&entity.Chat{}, &entity.ChatSession{}, &entity.ConversationMessage{}, &entity.ConversationReference{},
&entity.API4Conversation{}, &entity.API4ConversationMessage{}, &entity.API4ConversationReference{},
&entity.ChatChannel{}, &entity.UserCanvas{}, &entity.UserCanvasVersion{},
&entity.Search{}, &entity.Memory{}, &entity.MemoryTask{}, &entity.MCPServer{},
&entity.SkillSpace{}, &entity.SkillSearchConfig{},
&entity.CompilationTemplate{}, &entity.CompilationTemplateGroup{},
&entity.EvaluationDataset{}, &entity.EvaluationCase{}, &entity.EvaluationRun{}, &entity.EvaluationResult{},
&entity.Connector{}, &entity.Connector2Kb{}, &entity.SyncLogs{},
&entity.KnowledgeCompileDataset{}, &entity.WikiDocumentDirty{}, &entity.PipelineOperationLog{},
&entity.IngestionTask{}, &entity.IngestionTaskLog{}, &entity.FileCommit{}, &entity.FileCommitItem{},
&entity.TenantModelProvider{}, &entity.TenantModelInstance{}, &entity.TenantModel{},
&entity.TenantModelGroup{}, &entity.TenantModelGroupMapping{},
&entity.TenantLLM{}, &entity.TenantLangfuse{}, &entity.APIToken{}, &entity.InvitationCode{},
)
t.Cleanup(testutil.ReplaceDBForTest(t, db))
return db
}
func TestDeleteUserRemovesOwnedDataAndJoinedDocuments(t *testing.T) {
db := setupUserDeletionDB(t)
insert := func(query string, args ...interface{}) {
t.Helper()
if err := db.Exec(query, args...).Error; err != nil {
t.Fatalf("insert fixture: %v", err)
}
}
insert("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0")
insert("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-2", "[email protected]", "Other", "1", "1", "0")
insert("INSERT INTO tenant (id, llm_id, embd_id, asr_id, img2txt_id, rerank_id, parser_ids) VALUES (?, ?, ?, ?, ?, ?, ?)", "user-1", "llm", "embd", "asr", "img", "rerank", "general")
insert("INSERT INTO user_tenant (id, user_id, tenant_id, role, invited_by) VALUES (?, ?, ?, ?, ?)", "owner", "user-1", "user-1", "owner", "user-1")
insert("INSERT INTO user_tenant (id, user_id, tenant_id, role, invited_by) VALUES (?, ?, ?, ?, ?)", "joined", "user-1", "team", "normal", "user-2")
insert("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id, doc_num, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "own-kb", "user-1", "Own", "embd", "me", "user-1", "general", 1, 2, 3)
insert("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id, doc_num, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "team-kb", "team", "Team", "embd", "team", "user-2", "general", 2, 5, 7)
insert("INSERT INTO document (id, kb_id, name, parser_id, parser_config, type, created_by, suffix, location, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "own-doc", "own-kb", "Owned document", "general", "{}", "pdf", "user-1", "pdf", "own.pdf", 2, 3)
insert("INSERT INTO document (id, kb_id, name, parser_id, parser_config, type, created_by, suffix, location, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "joined-doc", "team-kb", "Joined document", "general", "{}", "pdf", "user-1", "pdf", "joined.pdf", 2, 3)
insert("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", "other-doc", "team-kb", "general", "{}", "pdf", "user-2", "pdf", 3, 4)
insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, location, type) VALUES (?, ?, ?, ?, ?, ?, ?)", "own-file", "own-folder", "user-1", "user-1", "own.pdf", "own.pdf", "file")
insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, type) VALUES (?, ?, ?, ?, ?, ?)", "own-folder", "own-folder", "user-1", "user-1", "/", "folder")
insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, location, type, source_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "kb-file", "kb-folder", "user-1", "user-1", "own.pdf", "own.pdf", "pdf", "knowledgebase")
insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, location, type) VALUES (?, ?, ?, ?, ?, ?, ?)", "joined-file", "team-folder", "team", "user-1", "joined.pdf", "joined.pdf", "file")
insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, location, type) VALUES (?, ?, ?, ?, ?, ?, ?)", "other-file", "team-folder", "team", "user-2", "other.pdf", "other.pdf", "file")
insert("INSERT INTO file2document (id, file_id, document_id) VALUES (?, ?, ?)", "own-link", "own-file", "own-doc")
insert("INSERT INTO file2document (id, file_id, document_id) VALUES (?, ?, ?)", "joined-link", "joined-file", "joined-doc")
insert("INSERT INTO file2document (id, file_id, document_id) VALUES (?, ?, ?)", "shared-link", "other-file", "joined-doc")
insert("INSERT INTO file2document (id, file_id, document_id) VALUES (?, ?, ?)", "other-link", "other-file", "other-doc")
insert("INSERT INTO search (id, tenant_id, name, created_by, search_config) VALUES (?, ?, ?, ?, ?)", "own-search", "user-1", "Own", "user-1", "{}")
insert("INSERT INTO search (id, tenant_id, name, created_by, search_config) VALUES (?, ?, ?, ?, ?)", "joined-search", "team", "Joined", "user-1", "{}")
insert("INSERT INTO search (id, tenant_id, name, created_by, search_config) VALUES (?, ?, ?, ?, ?)", "other-search", "team", "Other", "user-2", "{}")
insert("INSERT INTO dialog (id, tenant_id, llm_id, llm_setting, prompt_config, do_refer, kb_ids) VALUES (?, ?, ?, ?, ?, ?, ?)", "own-chat", "user-1", "llm", "{}", "{}", "1", "[]")
insert("INSERT INTO dialog (id, tenant_id, llm_id, llm_setting, prompt_config, do_refer, kb_ids) VALUES (?, ?, ?, ?, ?, ?, ?)", "team-chat", "team", "llm", "{}", "{}", "1", "[]")
insert("INSERT INTO conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "own-conversation", "own-chat", "user-1")
insert("INSERT INTO conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "joined-conversation", "team-chat", "user-1")
insert("INSERT INTO conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "other-conversation", "team-chat", "user-2")
insert("INSERT INTO conversation_message (conversation_id, position, metadata) VALUES (?, ?, ?)", "own-conversation", 0, "{}")
insert("INSERT INTO conversation_reference (conversation_id, position, reference) VALUES (?, ?, ?)", "own-conversation", 0, "{}")
insert("INSERT INTO api_4_conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "own-api-conversation", "own-chat", "user-1")
insert("INSERT INTO api_4_conversation_message (conversation_id, position, metadata) VALUES (?, ?, ?)", "own-api-conversation", 0, "{}")
insert("INSERT INTO api_4_conversation_reference (conversation_id, position, reference) VALUES (?, ?, ?)", "own-api-conversation", 0, "{}")
insert("INSERT INTO user_canvas (id, user_id, tags, canvas_category) VALUES (?, ?, ?, ?)", "agent", "user-1", "", "agent_canvas")
insert("INSERT INTO user_canvas_version (id, user_canvas_id) VALUES (?, ?)", "agent-version", "agent")
insert("INSERT INTO api_4_conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "agent-conversation", "agent", "external-user")
insert("INSERT INTO chat_channel (id, tenant_id, name, channel, config) VALUES (?, ?, ?, ?, ?)", "channel", "user-1", "Channel", "telegram", "{}")
insert("INSERT INTO memory (id, tenant_id, name, embd_id, llm_id, permissions, forgetting_policy) VALUES (?, ?, ?, ?, ?, ?, ?)", "memory", "user-1", "Memory", "embd", "llm", "me", "FIFO")
insert("INSERT INTO tenant_model_provider (id, provider_name, tenant_id) VALUES (?, ?, ?)", "provider", "custom", "user-1")
insert("INSERT INTO tenant_model_instance (id, instance_name, provider_id, api_key) VALUES (?, ?, ?, ?)", "instance", "custom", "provider", "secret")
insert("INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type) VALUES (?, ?, ?, ?, ?)", "model", "model", "provider", "instance", 1)
insert("INSERT INTO tenant_model_group (id, group_type, strategy) VALUES (?, ?, ?)", "group", "chat", "weighted")
insert("INSERT INTO tenant_model_group_mapping (group_id, provider_id, instance_id, model_id) VALUES (?, ?, ?, ?)", "group", "provider", "instance", "model")
docEngine := &deletionEngine{}
store := &deletionStorage{}
core, logs := observer.New(zapcore.InfoLevel)
oldLogger := common.Logger
common.Logger = zap.New(core)
t.Cleanup(func() { common.Logger = oldLogger })
service := NewService()
service.deleteEngine = docEngine
service.deleteStorage = store
result, err := service.DeleteUser(t.Context(), "[email protected]")
if err != nil {
t.Fatalf("DeleteUser: %v", err)
}
if result.UserCount != 1 || result.TenantCount != 1 {
t.Fatalf("unexpected deletion result: %+v", result)
}
for _, item := range [][2]string{
{"knowledgebase", "own-kb"}, {"document", "joined-doc"}, {"file", "own-file"}, {"file", "joined-file"},
{"search", "own-search"}, {"search", "joined-search"}, {"dialog", "own-chat"},
{"conversation", "own-conversation"}, {"conversation", "joined-conversation"}, {"api_4_conversation", "own-api-conversation"}, {"api_4_conversation", "agent-conversation"},
{"user_canvas", "agent"}, {"user_canvas_version", "agent-version"}, {"chat_channel", "channel"},
{"memory", "memory"}, {"tenant_model_provider", "provider"}, {"tenant_model_instance", "instance"},
{"tenant_model", "model"}, {"tenant_model_group", "group"},
} {
var count int64
if err := db.Table(item[0]).Where("id = ?", item[1]).Count(&count).Error; err != nil {
t.Fatal(err)
}
if count != 0 {
t.Fatalf("%s retained deleted resource %s", item[0], item[1])
}
}
for _, table := range []string{"conversation_message", "conversation_reference", "api_4_conversation_message", "api_4_conversation_reference", "file2document", "tenant_model_group_mapping"} {
var count int64
want := int64(0)
if table == "file2document" {
want = 1
}
if err := db.Table(table).Count(&count).Error; err != nil || count != want {
t.Fatalf("dependent rows remain in %s: count=%d, err=%v", table, count, err)
}
}
for _, tableID := range [][2]string{{"document", "other-doc"}, {"file", "other-file"}, {"search", "other-search"}, {"knowledgebase", "team-kb"}, {"dialog", "team-chat"}, {"conversation", "other-conversation"}} {
var count int64
if err := db.Table(tableID[0]).Where("id = ?", tableID[1]).Count(&count).Error; err != nil || count != 1 {
t.Fatalf("shared resource %s/%s missing: count=%d, err=%v", tableID[0], tableID[1], count, err)
}
}
var team entity.Knowledgebase
if err := db.First(&team, "id = ?", "team-kb").Error; err != nil || team.DocNum != 1 || team.TokenNum != 3 || team.ChunkNum != 4 {
t.Fatalf("joined dataset counters = %+v, err=%v", team, err)
}
if !slices.Equal(store.buckets, []string{"own-kb", "own-folder"}) || !slices.Equal(store.files, []string{"own-kb/own.pdf", "team-kb/joined.pdf", "own-folder/own.pdf", "team-folder/joined.pdf"}) {
t.Fatalf("storage cleanup = buckets %v, files %v", store.buckets, store.files)
}
if !slices.Contains(docEngine.dropped, "ragflow_user-1") || !slices.Contains(docEngine.dropped, "memory_user-1") || !slices.Contains(docEngine.deleted, "doc_id:[joined-doc]") || !slices.Contains(docEngine.deleted, "metadata:id:[joined-doc]") {
t.Fatalf("index cleanup = dropped %v, deleted %v", docEngine.dropped, docEngine.deleted)
}
for _, check := range []struct {
message string
field string
want string
}{
{"Removed document object", "document", "Joined document (joined-doc)"},
{"Removed document object", "dataset", "Team (team-kb)"},
{"Removed file object", "file", "own.pdf (own-file)"},
{"Removed empty folder bucket", "folder", "/ (own-folder)"},
} {
matched := false
for _, entry := range logs.FilterMessage(check.message).All() {
if entry.ContextMap()[check.field] == check.want {
matched = true
break
}
}
if !matched {
t.Errorf("missing %s log with %s=%q", check.message, check.field, check.want)
}
}
}
func TestDeleteUserKeepsUserWhenExternalCleanupFails(t *testing.T) {
db := setupUserDeletionDB(t)
if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
t.Fatal(err)
}
if err := db.Exec("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id) VALUES (?, ?, ?, ?, ?, ?, ?)", "dataset", "team", "Dataset", "embd", "me", "user-1", "general").Error; err != nil {
t.Fatal(err)
}
if err := db.Exec("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "doc", "dataset", "general", "{}", "pdf", "user-1", "pdf", "doc.pdf").Error; err != nil {
t.Fatal(err)
}
service := NewService()
service.deleteEngine = &deletionEngine{}
service.deleteStorage = &deletionStorage{err: errors.New("storage unavailable")}
if _, err := service.DeleteUser(t.Context(), "[email protected]"); err == nil || !strings.Contains(err.Error(), "storage unavailable") {
t.Fatalf("DeleteUser error = %v", err)
}
var count int64
if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 1 {
t.Fatalf("user was deleted after external failure: count=%d, err=%v", count, err)
}
}
func TestDeleteUserContinuesWhenEmptyBucketRemovalFails(t *testing.T) {
db := setupUserDeletionDB(t)
if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
t.Fatal(err)
}
if err := db.Exec("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id) VALUES (?, ?, ?, ?, ?, ?, ?)", "dataset", "team", "Dataset", "embd", "me", "user-1", "general").Error; err != nil {
t.Fatal(err)
}
service := NewService()
service.deleteEngine = &deletionEngine{}
service.deleteStorage = &deletionStorage{bucketErr: errors.New("bucket is not empty")}
if _, err := service.DeleteUser(t.Context(), "[email protected]"); err != nil {
t.Fatalf("DeleteUser after bucket removal failure: %v", err)
}
var count int64
if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 0 {
t.Fatalf("user retained after bucket failure: count=%d, err=%v", count, err)
}
}
func TestDeleteUserContinuesWhenStorageItemsAreMissing(t *testing.T) {
db := setupUserDeletionDB(t)
if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
t.Fatal(err)
}
if err := db.Exec("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id) VALUES (?, ?, ?, ?, ?, ?, ?)", "dataset", "team", "Dataset", "embd", "me", "user-1", "general").Error; err != nil {
t.Fatal(err)
}
if err := db.Exec("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "doc", "dataset", "general", "{}", "pdf", "user-1", "pdf", "doc.pdf").Error; err != nil {
t.Fatal(err)
}
store := &deletionStorage{missing: true}
service := NewService()
service.deleteEngine = &deletionEngine{}
service.deleteStorage = store
if _, err := service.DeleteUser(t.Context(), "[email protected]"); err != nil {
t.Fatalf("DeleteUser with missing storage items: %v", err)
}
if len(store.files) != 0 || len(store.buckets) != 0 {
t.Fatalf("storage deletion attempted for missing items: files=%v buckets=%v", store.files, store.buckets)
}
}
func TestDeleteUserStopsOnStorageCheckError(t *testing.T) {
db := setupUserDeletionDB(t)
if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
t.Fatal(err)
}
if err := db.Exec("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id) VALUES (?, ?, ?, ?, ?, ?, ?)", "dataset", "team", "Dataset", "embd", "me", "user-1", "general").Error; err != nil {
t.Fatal(err)
}
if err := db.Exec("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "doc", "dataset", "general", "{}", "pdf", "user-1", "pdf", "doc.pdf").Error; err != nil {
t.Fatal(err)
}
service := NewService()
service.deleteEngine = &deletionEngine{}
service.deleteStorage = &deletionStorage{checkErr: errors.New("storage unavailable")}
if _, err := service.DeleteUser(t.Context(), "[email protected]"); err == nil || !strings.Contains(err.Error(), "storage unavailable") {
t.Fatalf("DeleteUser error = %v", err)
}
var count int64
if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 1 {
t.Fatalf("user deleted after storage check failed: count=%d, err=%v", count, err)
}
}
func TestDeleteUserRollsBackDatabaseCleanup(t *testing.T) {
db := setupUserDeletionDB(t)
if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
t.Fatal(err)
}
if err := db.Exec("INSERT INTO user_canvas (id, user_id, tags, canvas_category) VALUES (?, ?, ?, ?)", "agent", "user-1", "", "agent_canvas").Error; err != nil {
t.Fatal(err)
}
if err := db.Exec("INSERT INTO user_canvas_version (id, user_canvas_id) VALUES (?, ?)", "version", "agent").Error; err != nil {
t.Fatal(err)
}
if err := db.Migrator().DropTable(&entity.Search{}); err != nil {
t.Fatal(err)
}
if _, err := NewService().DeleteUser(t.Context(), "[email protected]"); err == nil {
t.Fatal("expected SQL cleanup failure")
}
for _, item := range [][2]string{{"user", "user-1"}, {"user_canvas", "agent"}, {"user_canvas_version", "version"}} {
var count int64
if err := db.Table(item[0]).Where("id = ?", item[1]).Count(&count).Error; err != nil || count != 1 {
t.Fatalf("transaction did not preserve %s/%s: count=%d, err=%v", item[0], item[1], count, err)
}
}
}
+9 -2
View File
@@ -1339,8 +1339,15 @@ func (s *chunkImageStorage) ListObjects(ctx context.Context, bucket string, tena
func (s *chunkImageStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
return "", nil
}
func (s *chunkImageStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
func (s *chunkImageStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
func (s *chunkImageStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
func (s *chunkImageStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
func (s *chunkImageStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error { return nil }
func (s *chunkImageStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
return s.exists, nil
}
func (s *chunkImageStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
return true, nil
}
func (s *chunkImageStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
return false
}
+51 -11
View File
@@ -318,19 +318,17 @@ func (d *DatasetService) DeleteDatasets(ctx context.Context, ids []string, delet
}
successCount := 0
errorsList := make([]string, 0)
for _, kb := range kbs {
if err := d.deleteDataset(ctx, tenantID, kb); err != nil {
errorsList = append(errorsList, err.Error())
common.Warn("deleteDataset failed", zap.String("kb_id", kb.ID), zap.Error(err))
continue
common.Warn("deleteDataset failed", zap.String("dataset", datasetNameAndID(kb)), zap.String("kb_id", kb.ID), zap.Error(err))
return nil, common.CodeServerError, err
}
successCount++
}
return map[string]interface{}{
"success_count": successCount,
"errors": errorsList,
"errors": []string{},
}, common.CodeSuccess, nil
}
@@ -339,6 +337,7 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
if storageImpl == nil {
return fmt.Errorf("storage not initialized")
}
dataset := datasetNameAndID(kb)
// Collect document IDs first so engine cleanup can run before the
// transaction (engine ops are not transactional).
@@ -346,6 +345,38 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
if err := dao.DB.Where("kb_id = ?", kb.ID).Find(&documents).Error; err != nil {
return fmt.Errorf("delete dataset error for %s", kb.ID)
}
cleanupCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
for _, document := range documents {
if document.Location == nil || *document.Location == "" {
continue
}
exists, err := storageImpl.ObjectExists(cleanupCtx, kb.ID, *document.Location)
if err != nil {
return fmt.Errorf("check document %s in dataset %s: %w", document.ID, kb.ID, err)
}
if !exists {
common.Warn("Dataset document object already missing", zap.String("document", documentNameAndID(document)), zap.String("document_id", document.ID), zap.String("dataset", dataset), zap.String("bucket", kb.ID))
continue
}
if err := storageImpl.Remove(cleanupCtx, kb.ID, *document.Location); err != nil {
return fmt.Errorf("remove document %s from dataset %s: %w", document.ID, kb.ID, err)
}
common.Info("Removed dataset document object", zap.String("document", documentNameAndID(document)), zap.String("document_id", document.ID), zap.String("dataset", dataset), zap.String("bucket", kb.ID))
}
exists, err := storageImpl.BucketExistsWithError(cleanupCtx, kb.ID)
if err != nil {
return fmt.Errorf("check dataset bucket %s: %w", kb.ID, err)
}
if !exists {
common.Warn("Dataset bucket already missing", zap.String("dataset", dataset), zap.String("bucket", kb.ID))
} else {
if err := storageImpl.RemoveEmptyBucket(cleanupCtx, kb.ID); err != nil {
common.Warn("Unable to remove empty dataset bucket", zap.String("dataset", dataset), zap.String("bucket", kb.ID), zap.Error(err))
} else {
common.Info("Removed empty dataset bucket", zap.String("dataset", dataset), zap.String("bucket", kb.ID))
}
}
docIDs := extractDocIDs(documents)
if len(docIDs) > 0 {
d.deleteDatasetEngineData(ctx, kb, docIDs)
@@ -397,15 +428,24 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
}); err != nil {
return err
}
// Dataset uploads and generated objects are stored under the KB ID.
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Minute)
defer cancel()
if err := storageImpl.RemoveBucket(cleanupCtx, kb.ID); err != nil {
return fmt.Errorf("dataset %s deleted but storage cleanup failed: %w", kb.ID, err)
}
common.Info("Deleted dataset", zap.String("dataset", dataset), zap.String("kb_id", kb.ID))
return nil
}
func documentNameAndID(document entity.Document) string {
if document.Name == nil || *document.Name == "" {
return document.ID
}
return fmt.Sprintf("%s (%s)", *document.Name, document.ID)
}
func datasetNameAndID(kb *entity.Knowledgebase) string {
if kb.Name == "" {
return kb.ID
}
return fmt.Sprintf("%s (%s)", kb.Name, kb.ID)
}
func (d *DatasetService) ListDatasets(ctx context.Context, id, name string, page, pageSize int, terms []dao.OrderTerm, keywords string, ownerIDs []string, parserID, userID string, ids []string) ([]map[string]interface{}, int64, common.ErrorCode, error) {
id = strings.TrimSpace(id)
if id != "" && len(ids) > 0 {
@@ -0,0 +1,103 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package dataset
import (
"testing"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/storage"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"go.uber.org/zap/zaptest/observer"
)
func TestDeleteDatasetLeavesNonemptyBucket(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Dataset")
if err := db.Exec("INSERT INTO document (id, kb_id, name, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", "doc-1", "kb-1", "Document", "general", "{}", "pdf", "tenant-1", "pdf", "tracked").Error; err != nil {
t.Fatal(err)
}
store := storage.NewMemoryStorage()
if err := store.Put(t.Context(), "kb-1", "tracked", []byte("content")); err != nil {
t.Fatal(err)
}
if err := store.Put(t.Context(), "kb-1", "untracked", []byte("content")); err != nil {
t.Fatal(err)
}
factory := storage.GetStorageFactory()
previous := factory.GetStorage()
factory.SetStorage(store)
t.Cleanup(func() { factory.SetStorage(previous) })
core, logs := observer.New(zapcore.InfoLevel)
previousLogger := common.Logger
common.Logger = zap.New(core)
t.Cleanup(func() { common.Logger = previousLogger })
kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
if err != nil {
t.Fatal(err)
}
service := testDatasetUpdateService(t)
if err := service.deleteDataset(t.Context(), "tenant-1", kb); err != nil {
t.Fatalf("delete dataset with nonempty bucket: %v", err)
}
if !store.ObjExist(t.Context(), "kb-1", "untracked") {
t.Fatal("dataset deletion removed an untracked object")
}
if store.ObjExist(t.Context(), "kb-1", "tracked") {
t.Fatal("dataset deletion retained a document object")
}
var count int64
if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Count(&count).Error; err != nil || count != 0 {
t.Fatalf("dataset retained after storage failure: count=%d, err=%v", count, err)
}
if entries := logs.FilterMessage("Removed dataset document object").All(); len(entries) != 1 || entries[0].ContextMap()["document"] != "Document (doc-1)" || entries[0].ContextMap()["dataset"] != "Dataset (kb-1)" {
t.Errorf("document deletion logs = %v", entries)
}
if entries := logs.FilterMessage("Deleted dataset").All(); len(entries) != 1 || entries[0].ContextMap()["dataset"] != "Dataset (kb-1)" {
t.Errorf("dataset deletion logs = %v", entries)
}
}
func TestDeleteDatasetContinuesWhenStorageIsMissing(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Dataset")
if err := db.Exec("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "doc-1", "kb-1", "general", "{}", "pdf", "tenant-1", "pdf", "missing").Error; err != nil {
t.Fatal(err)
}
factory := storage.GetStorageFactory()
previous := factory.GetStorage()
factory.SetStorage(storage.NewMemoryStorage())
t.Cleanup(func() { factory.SetStorage(previous) })
kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
if err != nil {
t.Fatal(err)
}
if err := testDatasetUpdateService(t).deleteDataset(t.Context(), "tenant-1", kb); err != nil {
t.Fatalf("delete dataset with missing storage: %v", err)
}
var count int64
if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Count(&count).Error; err != nil || count != 0 {
t.Fatalf("dataset retained after missing storage: count=%d, err=%v", count, err)
}
}
+9 -2
View File
@@ -123,8 +123,15 @@ func (f *fakeUploadStorage) ListObjects(ctx context.Context, bucket string, tena
func (f *fakeUploadStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
return "", nil
}
func (f *fakeUploadStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
func (f *fakeUploadStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
func (f *fakeUploadStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
func (f *fakeUploadStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
func (f *fakeUploadStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error { return nil }
func (f *fakeUploadStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
return f.ObjExist(ctx, bucket, fnm), nil
}
func (f *fakeUploadStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
return true, nil
}
func (f *fakeUploadStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
v, ok := f.objects[f.key(srcBucket, srcPath)]
if !ok {
+84 -23
View File
@@ -2,12 +2,14 @@ package file
import (
"context"
"errors"
"fmt"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/storage"
"go.uber.org/zap"
)
// DeleteFiles deletes files by IDs
@@ -58,17 +60,36 @@ func (s *FileService) DeleteFiles(ctx context.Context, uid string, fileIDs []str
// deleteSingleFile deletes a single file (not folder)
// Matches Python's _delete_single_file function
func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) error {
// 1. Delete storage object
if file.Location != nil && *file.Location != "" {
storageImpl := storage.GetStorageFactory().GetStorage()
if storageImpl != nil {
if err := storageImpl.Remove(ctx, file.ParentID, *file.Location); err != nil {
common.Logger.Error(fmt.Sprintf("Fail to remove object: %s/%s, error: %v", file.ParentID, *file.Location, err))
}
}
storageImpl := storage.GetStorageFactory().GetStorage()
if err := removeFileObject(ctx, storageImpl, file); err != nil {
return err
}
return s.deleteSingleFileRecords(ctx, file)
}
// 2. Handle associated documents
func removeFileObject(ctx context.Context, storageImpl storage.Storage, file *entity.File) error {
if file.Location != nil && *file.Location != "" {
if storageImpl == nil {
return fmt.Errorf("storage is not configured for file %s", file.ID)
}
exists, err := storageImpl.ObjectExists(ctx, file.ParentID, *file.Location)
if err != nil {
return fmt.Errorf("check file %s: %w", file.ID, err)
}
if !exists {
common.Warn("File object already missing", zap.String("file", fileNameAndID(file)), zap.String("file_id", file.ID), zap.String("bucket", file.ParentID))
return nil
}
if err := storageImpl.Remove(ctx, file.ParentID, *file.Location); err != nil {
return fmt.Errorf("remove file %s: %w", file.ID, err)
}
common.Info("Removed file object", zap.String("file", fileNameAndID(file)), zap.String("file_id", file.ID), zap.String("bucket", file.ParentID))
}
return nil
}
func (s *FileService) deleteSingleFileRecords(ctx context.Context, file *entity.File) error {
// Handle associated documents
informs, err := s.file2DocumentDAO.GetByFileID(ctx, dao.DB, file.ID)
if err != nil {
return fmt.Errorf("failed to get file2document mappings: %w", err)
@@ -81,10 +102,7 @@ func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) e
docID := *inform.DocumentID
if s.documentService != nil {
if err = s.documentService.RemoveDocumentKeepFile(ctx, docID); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return fmt.Errorf("context cancelled while removing document %s: %w", docID, err)
}
common.Logger.Error(fmt.Sprintf("Fail to remove document: %s, error: %v", docID, err))
return fmt.Errorf("remove document %s: %w", docID, err)
}
}
}
@@ -95,10 +113,11 @@ func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) e
}
}
// 3. Delete file record
// Delete file record
if err = s.fileDAO.Delete(ctx, dao.DB, file.ID); err != nil {
return err
}
common.Info("Deleted file", zap.String("file", fileNameAndID(file)), zap.String("file_id", file.ID))
return nil
}
@@ -106,30 +125,72 @@ func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) e
// deleteFolderRecursive recursively deletes a folder and its contents
// Matches Python's _delete_folder_recursive function
func (s *FileService) deleteFolderRecursive(ctx context.Context, folder *entity.File, uid string) error {
// Get all sub-files
storageImpl := storage.GetStorageFactory().GetStorage()
if storageImpl == nil {
return fmt.Errorf("storage is not configured for folder %s", folder.ID)
}
if err := s.removeFolderObjectsRecursive(ctx, folder, storageImpl); err != nil {
return err
}
return s.deleteFolderRecordsRecursive(ctx, folder)
}
func (s *FileService) removeFolderObjectsRecursive(ctx context.Context, folder *entity.File, storageImpl storage.Storage) error {
subFiles, err := s.fileDAO.ListByParentID(ctx, dao.DB, folder.ID)
if err != nil {
return err
}
for _, subFile := range subFiles {
if subFile.Type == FileTypeFolder {
// Recursively delete subfolder
if err = s.deleteFolderRecursive(ctx, subFile, uid); err != nil {
if err = s.removeFolderObjectsRecursive(ctx, subFile, storageImpl); err != nil {
return err
}
} else {
// Delete single file
if err = s.deleteSingleFile(ctx, subFile); err != nil {
if err = removeFileObject(ctx, storageImpl, subFile); err != nil {
return err
}
}
}
exists, err := storageImpl.BucketExistsWithError(ctx, folder.ID)
if err != nil {
return fmt.Errorf("check folder bucket %s: %w", folder.ID, err)
}
if !exists {
common.Warn("Folder bucket already missing", zap.String("folder", fileNameAndID(folder)), zap.String("bucket", folder.ID))
return nil
}
if err := storageImpl.RemoveEmptyBucket(ctx, folder.ID); err != nil {
common.Warn("Unable to remove empty folder bucket", zap.String("folder", fileNameAndID(folder)), zap.String("bucket", folder.ID), zap.Error(err))
} else {
common.Info("Removed empty folder bucket", zap.String("folder", fileNameAndID(folder)), zap.String("bucket", folder.ID))
}
return nil
}
// Delete the folder itself
func (s *FileService) deleteFolderRecordsRecursive(ctx context.Context, folder *entity.File) error {
subFiles, err := s.fileDAO.ListByParentID(ctx, dao.DB, folder.ID)
if err != nil {
return err
}
for _, subFile := range subFiles {
if subFile.Type == FileTypeFolder {
if err = s.deleteFolderRecordsRecursive(ctx, subFile); err != nil {
return err
}
} else if err = s.deleteSingleFileRecords(ctx, subFile); err != nil {
return err
}
}
if err = s.fileDAO.Delete(ctx, dao.DB, folder.ID); err != nil {
return err
}
common.Info("Deleted folder", zap.String("folder", fileNameAndID(folder)), zap.String("folder_id", folder.ID))
return nil
}
func fileNameAndID(file *entity.File) string {
if file.Name == "" {
return file.ID
}
return fmt.Sprintf("%s (%s)", file.Name, file.ID)
}
+119
View File
@@ -0,0 +1,119 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package file
import (
"context"
"errors"
"testing"
"ragflow/internal/common"
"ragflow/internal/storage"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"go.uber.org/zap/zaptest/observer"
)
type failingObjectCheckStorage struct{ storage.Storage }
func (f failingObjectCheckStorage) ObjectExists(context.Context, string, string) (bool, error) {
return false, errors.New("storage unavailable")
}
func TestDeleteFolderLeavesNonemptyBucket(t *testing.T) {
db := setupFolderTestDB(t)
folder, err := testFileService().fileDAO.CreateFolder(t.Context(), db, "root", "tenant-1", "folder", FileTypeFolder)
if err != nil {
t.Fatal(err)
}
insertFolderTestFile(t, "file-1", folder.ID, "tracked")
store := storage.NewMemoryStorage()
if err := store.Put(t.Context(), folder.ID, "tracked", []byte("content")); err != nil {
t.Fatal(err)
}
if err := store.Put(t.Context(), folder.ID, "untracked", []byte("content")); err != nil {
t.Fatal(err)
}
factory := storage.GetStorageFactory()
previous := factory.GetStorage()
factory.SetStorage(store)
t.Cleanup(func() { factory.SetStorage(previous) })
core, logs := observer.New(zapcore.InfoLevel)
previousLogger := common.Logger
common.Logger = zap.New(core)
t.Cleanup(func() { common.Logger = previousLogger })
service := testFileService()
if err := service.deleteFolderRecursive(t.Context(), folder, "tenant-1"); err != nil {
t.Fatalf("delete folder with nonempty bucket: %v", err)
}
if !store.ObjExist(t.Context(), folder.ID, "untracked") {
t.Fatal("folder deletion removed an untracked object")
}
if store.ObjExist(t.Context(), folder.ID, "tracked") {
t.Fatal("folder deletion retained a file object")
}
if _, err := service.fileDAO.GetByID(t.Context(), db, folder.ID); err == nil {
t.Fatal("folder record retained after storage failure")
}
if entries := logs.FilterMessage("Deleted file").All(); len(entries) != 1 || entries[0].ContextMap()["file"] != "tracked (file-1)" {
t.Errorf("file deletion logs = %v", entries)
}
if entries := logs.FilterMessage("Deleted folder").All(); len(entries) != 1 || entries[0].ContextMap()["folder"] != "folder ("+folder.ID+")" {
t.Errorf("folder deletion logs = %v", entries)
}
}
func TestDeleteFolderStopsOnStorageCheckError(t *testing.T) {
db := setupFolderTestDB(t)
folder, err := testFileService().fileDAO.CreateFolder(t.Context(), db, "root", "tenant-1", "folder", FileTypeFolder)
if err != nil {
t.Fatal(err)
}
insertFolderTestFile(t, "file-1", folder.ID, "tracked")
factory := storage.GetStorageFactory()
previous := factory.GetStorage()
factory.SetStorage(failingObjectCheckStorage{storage.NewMemoryStorage()})
t.Cleanup(func() { factory.SetStorage(previous) })
if err := testFileService().deleteFolderRecursive(t.Context(), folder, "tenant-1"); err == nil {
t.Fatal("expected storage check error")
}
if _, err := testFileService().fileDAO.GetByID(t.Context(), db, "file-1"); err != nil {
t.Fatalf("file record deleted after storage check failed: %v", err)
}
}
func TestDeleteFolderContinuesWhenStorageIsMissing(t *testing.T) {
db := setupFolderTestDB(t)
service := testFileService()
folder, err := service.fileDAO.CreateFolder(t.Context(), db, "root", "tenant-1", "folder", FileTypeFolder)
if err != nil {
t.Fatal(err)
}
insertFolderTestFile(t, "file-1", folder.ID, "missing")
factory := storage.GetStorageFactory()
previous := factory.GetStorage()
factory.SetStorage(storage.NewMemoryStorage())
t.Cleanup(func() { factory.SetStorage(previous) })
if err := service.deleteFolderRecursive(t.Context(), folder, "tenant-1"); err != nil {
t.Fatalf("delete folder with missing storage: %v", err)
}
if _, err := service.fileDAO.GetByID(t.Context(), db, folder.ID); err == nil {
t.Fatal("folder record retained after missing storage")
}
}
+10
View File
@@ -123,6 +123,16 @@ func (f *fakeStorage) RemoveBucket(ctx context.Context, bucket string) error {
panic("not implemented in fakeStorage")
}
func (f *fakeStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
panic("not implemented in fakeStorage")
}
func (f *fakeStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
return f.ObjExist(ctx, bucket, fnm), nil
}
func (f *fakeStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
panic("not implemented in fakeStorage")
}
func (f *fakeStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
panic("not implemented in fakeStorage")
}
+7 -6
View File
@@ -922,7 +922,7 @@ func (s *MemoryService) DeleteMemory(ctx context.Context, userID, memoryID strin
// TODO: Delete associated message index - Implementation pending MessageService
if s.docEngine != nil && engine.IsOceanBaseFamily(s.docEngine.GetType()) {
if err := s.docEngine.DropChunkStore(ctx, memoryIndexName(memory.TenantID), memoryID); err != nil {
if err := s.docEngine.DropChunkStore(ctx, MemoryIndexName(memory.TenantID), memoryID); err != nil {
return fmt.Errorf("delete memory messages: %w", err)
}
}
@@ -964,7 +964,7 @@ func (s *MemoryService) ForgetMessage(ctx context.Context, userID string, memory
condition := map[string]interface{}{
"id": messageDocID,
}
indexName := memoryIndexName(memory.TenantID)
indexName := MemoryIndexName(memory.TenantID)
if err = s.docEngine.UpdateChunks(ctx, condition, updates, indexName, memoryID); err != nil {
if isMessageDocumentNotFound(err) {
@@ -1097,7 +1097,7 @@ func (s *MemoryService) UpdateMessageStatus(ctx context.Context, userID, memoryI
condition := map[string]interface{}{
"id": messageDocID,
}
indexName := memoryIndexName(memory.TenantID)
indexName := MemoryIndexName(memory.TenantID)
if err = s.docEngine.UpdateChunks(ctx, condition, updates, indexName, memoryID); err != nil {
if isMessageDocumentNotFound(err) {
return false, &ResourceNotFoundError{Resource: "Message", ID: messageDocID}
@@ -1121,7 +1121,7 @@ func (s *MemoryService) GetMessageContent(ctx context.Context, userID, memoryID
return nil, errors.New("message store is not initialized")
}
indexName := memoryIndexName(memory.TenantID)
indexName := MemoryIndexName(memory.TenantID)
docID := fmt.Sprintf("%s_%d", memoryID, messageID)
res, err := s.docEngine.GetChunk(ctx, indexName, docID, []string{memoryID})
if err != nil {
@@ -1423,7 +1423,8 @@ func memoryMessageSelectFields() []string {
}
}
func memoryIndexName(tenantID string) string {
// MemoryIndexName returns the message index shared by a tenant's memories.
func MemoryIndexName(tenantID string) string {
prefix := strings.TrimSpace(common.GetEnv(common.EnvESIndexPrefix))
if prefix == "" {
return fmt.Sprintf("memory_%s", tenantID)
@@ -1438,7 +1439,7 @@ func memorySearchIndexNames(memories []*entity.Memory) []string {
if memory == nil {
continue
}
indexName := memoryIndexName(memory.TenantID)
indexName := MemoryIndexName(memory.TenantID)
if engine.GetEngineType() == "infinity" {
indexName = fmt.Sprintf("%s_%s", indexName, memory.ID)
}
+1 -1
View File
@@ -353,7 +353,7 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Cr
message["doc_id"] = message["memory_id"]
}
indexName := memoryIndexName(mem.TenantID)
indexName := MemoryIndexName(mem.TenantID)
exists, err := s.memories.docEngine.ChunkStoreExists(ctx, indexName, mem.ID)
if err != nil {
return fmt.Errorf("check message index: %w", err)
+4 -4
View File
@@ -34,12 +34,12 @@ func TestIsMessageDocumentNotFound(t *testing.T) {
func TestMemoryIndexNameMatchesPythonPrefix(t *testing.T) {
t.Setenv(common.EnvESIndexPrefix, "")
if got := memoryIndexName("tenant-1"); got != "memory_tenant-1" {
t.Fatalf("memoryIndexName() = %q", got)
if got := MemoryIndexName("tenant-1"); got != "memory_tenant-1" {
t.Fatalf("MemoryIndexName() = %q", got)
}
t.Setenv(common.EnvESIndexPrefix, "legacy")
if got := memoryIndexName("tenant-1"); got != "memory_legacy_tenant-1" {
t.Fatalf("memoryIndexName() with prefix = %q", got)
if got := MemoryIndexName("tenant-1"); got != "memory_legacy_tenant-1" {
t.Fatalf("MemoryIndexName() with prefix = %q", got)
}
}
+7 -7
View File
@@ -168,7 +168,7 @@ func (s *SkillIndexerService) IndexSkill(ctx context.Context, tenantID, spaceID
}
}
indexName := getSkillIndexName(tenantID, spaceID)
indexName := SkillIndexName(tenantID, spaceID)
// For Infinity: ensure table exists with correct dimension BEFORE inserting
if docEngine.GetType() == "infinity" {
@@ -249,7 +249,7 @@ func (s *SkillIndexerService) BatchIndexSkills(ctx context.Context, tenantID, sp
}
// Ensure index exists with correct dimension
indexName := getSkillIndexName(tenantID, spaceID)
indexName := SkillIndexName(tenantID, spaceID)
if docEngine.GetType() == "infinity" {
// For Infinity: must ensure table exists with correct dimension BEFORE inserting
common.Info(fmt.Sprintf("Checking if index exists: %s", indexName))
@@ -362,7 +362,7 @@ func (s *SkillIndexerService) BatchIndexSkills(ctx context.Context, tenantID, sp
// Returns nil if the document doesn't exist (idempotent delete)
func (s *SkillIndexerService) DeleteSkillIndex(ctx context.Context, tenantID, spaceID, skillID string, docEngine engine.DocEngine) error {
spaceID = normalizeSpaceID(spaceID)
indexName := getSkillIndexName(tenantID, spaceID)
indexName := SkillIndexName(tenantID, spaceID)
// ES document ID cannot contain '/' - replace with '_'
docID := strings.ReplaceAll(skillID, "/", "_")
if err := docEngine.DeleteDocument(ctx, indexName, docID); err != nil {
@@ -381,7 +381,7 @@ func (s *SkillIndexerService) DeleteSkillIndex(ctx context.Context, tenantID, sp
// Deletes all versions: both new format (skillname) and old format (skillname_x.x.x)
func (s *SkillIndexerService) DeleteSkillByName(ctx context.Context, tenantID, spaceID, skillName string, docEngine engine.DocEngine) error {
spaceID = normalizeSpaceID(spaceID)
indexName := getSkillIndexName(tenantID, spaceID)
indexName := SkillIndexName(tenantID, spaceID)
docID := strings.ReplaceAll(skillName, "/", "_")
if err := docEngine.DeleteDocument(ctx, indexName, docID); err != nil {
@@ -436,7 +436,7 @@ func (s *SkillIndexerService) ReindexAll(ctx context.Context, tenantID, spaceID
common.Info(fmt.Sprintf("ReindexAll: new embedding dimension is %d", newDimension))
// Delete existing index and recreate with new dimension (for both ES and Infinity)
indexName := getSkillIndexName(tenantID, spaceID)
indexName := SkillIndexName(tenantID, spaceID)
exists, _ := docEngine.ChunkStoreExists(ctx, indexName, "skill")
if exists {
common.Info(fmt.Sprintf("ReindexAll: deleting existing index %s", indexName))
@@ -843,7 +843,7 @@ func (s *SkillIndexerService) cleanupOldVersions(ctx context.Context, tenantID,
// InitializeIndex initializes the skill search index for a tenant
func (s *SkillIndexerService) InitializeIndex(ctx context.Context, tenantID, spaceID string, docEngine engine.DocEngine, embdID string) error {
// Check if index exists
indexName := getSkillIndexName(tenantID, spaceID)
indexName := SkillIndexName(tenantID, spaceID)
common.Info("Checking skill index existence", zap.String("indexName", indexName), zap.String("tenantID", tenantID), zap.String("spaceID", spaceID))
@@ -874,7 +874,7 @@ func (s *SkillIndexerService) createIndex(ctx context.Context, tenantID, spaceID
// createIndexWithDimension creates the skill index with a specific vector dimension
func (s *SkillIndexerService) createIndexWithDimension(ctx context.Context, tenantID, spaceID string, docEngine engine.DocEngine, embdID string, dimension int) error {
indexName := getSkillIndexName(tenantID, spaceID)
indexName := SkillIndexName(tenantID, spaceID)
common.Info(fmt.Sprintf("Creating skill index with dimension %d", dimension),
zap.String("indexName", indexName),
+3 -2
View File
@@ -222,7 +222,7 @@ func (s *SkillSearchService) Search(ctx context.Context, req *SearchRequest, doc
}
// Check if index exists before searching
indexName := getSkillIndexName(req.TenantID, req.SpaceID)
indexName := SkillIndexName(req.TenantID, req.SpaceID)
common.Debug("Searching skills", zap.String("indexName", indexName), zap.String("query", req.Query))
indexExists, err := docEngine.ChunkStoreExists(ctx, indexName, "skill")
@@ -694,7 +694,8 @@ func (s *SkillSearchService) getEmbedding(ctx context.Context, text, embdID, ten
}
// Helper functions
func getSkillIndexName(tenantID, spaceID string) string {
// SkillIndexName returns the index used by a tenant's skill space.
func SkillIndexName(tenantID, spaceID string) string {
spaceID = normalizeSpaceID(spaceID)
spaceID = strings.ToLower(spaceID)
replacer := strings.NewReplacer("-", "_", "/", "_", "\\", "_", " ", "_", ".", "_", ":", "_")
+1 -1
View File
@@ -439,7 +439,7 @@ func (s *SkillSpaceService) asyncDeleteSpace(ctx context.Context, spaceID, folde
// Step 1: Delete the search index
if docEngine != nil {
indexName := getSkillIndexName(tenantID, spaceID)
indexName := SkillIndexName(tenantID, spaceID)
common.Info("Async deleting space index", zap.String("index", indexName), zap.String("spaceID", spaceID))
deleteCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
if err := docEngine.DropChunkStore(deleteCtx, indexName, "skill"); err != nil {
+37
View File
@@ -111,6 +111,9 @@ func (m *GCSStorage) Remove(ctx context.Context, bucketName, objectName string,
obj := m.client.Bucket(bucketName).Object(objectName)
if err := obj.Delete(ctx); err != nil {
if errors.Is(err, storage.ErrObjectNotExist) || errors.Is(err, storage.ErrBucketNotExist) {
return nil
}
return fmt.Errorf("fail to delete object: %w", err)
}
@@ -130,6 +133,17 @@ func (m *GCSStorage) ObjExist(ctx context.Context, bucketName, objectName string
return true
}
func (m *GCSStorage) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, error) {
_, err := m.client.Bucket(bucketName).Object(objectName).Attrs(ctx)
if err == nil {
return true, nil
}
if errors.Is(err, storage.ErrObjectNotExist) || errors.Is(err, storage.ErrBucketNotExist) {
return false, nil
}
return false, err
}
func (m *GCSStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
bucketObject := m.client.Bucket(bucket)
@@ -182,6 +196,17 @@ func (m *GCSStorage) BucketExists(ctx context.Context, bucket string) bool {
return true
}
func (m *GCSStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
_, err := m.client.Bucket(bucket).Attrs(ctx)
if err == nil {
return true, nil
}
if errors.Is(err, storage.ErrBucketNotExist) {
return false, nil
}
return false, err
}
// RemoveBucket removes a bucket and all its objects
func (m *GCSStorage) RemoveBucket(ctx context.Context, bucketName string) error {
if bucketName == "" {
@@ -212,6 +237,18 @@ func (m *GCSStorage) RemoveBucket(ctx context.Context, bucketName string) error
return nil
}
// RemoveEmptyBucket removes a bucket only if GCS finds it empty.
func (m *GCSStorage) RemoveEmptyBucket(ctx context.Context, bucketName string) error {
if bucketName == "" {
return fmt.Errorf("attempt to delete bucket without name")
}
err := m.client.Bucket(bucketName).Delete(ctx)
if errors.Is(err, storage.ErrBucketNotExist) {
return nil
}
return err
}
// Copy copies an object from source to destination
func (m *GCSStorage) Copy(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) bool {
+31
View File
@@ -137,6 +137,16 @@ func (m *MemoryStorage) ObjExist(ctx context.Context, bucket, fnm string, tenant
return ok
}
func (m *MemoryStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
if err := ctx.Err(); err != nil {
return false, err
}
m.mu.RLock()
defer m.mu.RUnlock()
_, exists := m.objects[bucket][fnm]
return exists, nil
}
func (m *MemoryStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
m.mu.RLock()
defer m.mu.RUnlock()
@@ -183,6 +193,13 @@ func (m *MemoryStorage) BucketExists(ctx context.Context, bucket string) bool {
return ok
}
func (m *MemoryStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
if err := ctx.Err(); err != nil {
return false, err
}
return m.BucketExists(ctx, bucket), nil
}
// RemoveBucket deletes a bucket and all of its keys. Removing a
// non-existent bucket is a no-op and returns nil.
func (m *MemoryStorage) RemoveBucket(ctx context.Context, bucket string) error {
@@ -197,6 +214,20 @@ func (m *MemoryStorage) RemoveBucket(ctx context.Context, bucket string) error {
return nil
}
// RemoveEmptyBucket deletes a bucket only when it contains no objects.
func (m *MemoryStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
if err := ctx.Err(); err != nil {
return err
}
m.mu.Lock()
defer m.mu.Unlock()
if len(m.objects[bucket]) != 0 {
return fmt.Errorf("bucket %s is not empty", bucket)
}
delete(m.objects, bucket)
return nil
}
// Copy duplicates an object from srcBucket/srcKey to destBucket/destKey.
// The source is left untouched. Returns false if the source does not exist
// or if the destination bucket creation fails.
+23
View File
@@ -156,6 +156,29 @@ func TestMemoryStorage_RemoveBucket(t *testing.T) {
}
}
func TestMemoryStorage_RemoveEmptyBucket(t *testing.T) {
ms := newTestMemory(t)
ctx := t.Context()
if err := ms.Put(ctx, "bucket", "file", []byte("content")); err != nil {
t.Fatal(err)
}
if err := ms.RemoveEmptyBucket(ctx, "bucket"); err == nil {
t.Fatal("RemoveEmptyBucket removed a nonempty bucket")
}
if !ms.ObjExist(ctx, "bucket", "file") {
t.Fatal("RemoveEmptyBucket removed the file")
}
if err := ms.Remove(ctx, "bucket", "file"); err != nil {
t.Fatal(err)
}
if err := ms.RemoveEmptyBucket(ctx, "bucket"); err != nil {
t.Fatal(err)
}
if ms.BucketExists(ctx, "bucket") {
t.Fatal("RemoveEmptyBucket retained an empty bucket")
}
}
func TestMemoryStorage_CopyMove(t *testing.T) {
ms := newTestMemory(t)
ctx := t.Context()
+59
View File
@@ -234,6 +234,10 @@ func (m *MinioStorage) Remove(ctx context.Context, bucket, fnm string, tenantID
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
if err := m.client.RemoveObject(ctx, bucket, fnm, minio.RemoveObjectOptions{}); err != nil {
code := minio.ToErrorResponse(err).Code
if code == "NoSuchKey" || code == "NoSuchBucket" {
return nil
}
common.Warn("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
return err
}
@@ -263,6 +267,19 @@ func (m *MinioStorage) ObjExist(ctx context.Context, bucket, fnm string, tenantI
return true
}
func (m *MinioStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
_, err := m.client.StatObject(ctx, bucket, fnm, minio.StatObjectOptions{})
if err == nil {
return true, nil
}
code := minio.ToErrorResponse(err).Code
if code == "NoSuchKey" || code == "NoSuchBucket" {
return false, nil
}
return false, err
}
// GetPresignedURL generates a presigned URL for accessing an object
func (m *MinioStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
@@ -303,6 +320,13 @@ func (m *MinioStorage) BucketExists(ctx context.Context, bucket string) bool {
return exists
}
func (m *MinioStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
if m.bucket != "" {
bucket = m.bucket
}
return m.client.BucketExists(ctx, bucket)
}
func (m *MinioStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
var objects []string
@@ -401,6 +425,41 @@ func (m *MinioStorage) RemoveBucket(ctx context.Context, bucket string) error {
return nil
}
// RemoveEmptyBucket removes a bucket only when it contains no object versions.
func (m *MinioStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
actualBucket := bucket
prefix := ""
if m.bucket != "" {
actualBucket = m.bucket
if m.prefixPath != "" {
prefix = m.prefixPath + "/"
}
prefix += bucket + "/"
}
exists, err := m.client.BucketExists(ctx, actualBucket)
if err != nil {
return err
}
if !exists {
return nil
}
listCtx, cancel := context.WithCancel(ctx)
defer cancel()
for object := range m.client.ListObjects(listCtx, actualBucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true, WithVersions: true}) {
if object.Err != nil {
return object.Err
}
return fmt.Errorf("bucket %s is not empty", bucket)
}
if err := ctx.Err(); err != nil {
return err
}
if m.bucket == "" {
return m.client.RemoveBucket(ctx, actualBucket)
}
return nil
}
// Copy copies an object from source to destination
func (m *MinioStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
srcBucket, srcPath = m.resolveBucketAndPath(srcBucket, srcPath)
+44 -2
View File
@@ -253,6 +253,9 @@ func (o *OSSStorage) Remove(ctx context.Context, bucket, fnm string, tenantID ..
Key: aws.String(fnm),
})
if err != nil {
if isS3NotFound(err) {
return nil
}
common.Error("Failed to remove object", err, zap.String("bucket", bucket), zap.String("key", fnm))
return err
}
@@ -278,6 +281,18 @@ func (o *OSSStorage) ObjExist(ctx context.Context, bucket, fnm string, tenantID
return true
}
func (o *OSSStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
_, err := o.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(fnm)})
if err == nil {
return true, nil
}
if isS3NotFound(err) {
return false, nil
}
return false, err
}
func (o *OSSStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
listInput := &s3.ListObjectsV2Input{
@@ -344,13 +359,28 @@ func (o *OSSStorage) BucketExists(ctx context.Context, bucket string) bool {
return true
}
func (o *OSSStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
if o.bucket != "" {
bucket = o.bucket
}
_, err := o.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})
if err == nil {
return true, nil
}
if isS3BucketNotFound(err) {
return false, nil
}
return false, err
}
// RemoveBucket removes a bucket and all its objects
func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {
actualBucket := bucket
if o.bucket != "" {
actualBucket = o.bucket
return fmt.Errorf("cannot remove logical OSS bucket %s: shared bucket mode does not isolate objects by logical bucket", bucket)
}
actualBucket := bucket
// Check if bucket exists
if !o.BucketExists(ctx, actualBucket) {
return nil
@@ -396,6 +426,18 @@ func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {
return nil
}
// RemoveEmptyBucket removes a physical bucket only when it is empty.
func (o *OSSStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
if o.bucket != "" {
return fmt.Errorf("cannot remove logical OSS bucket %s: shared bucket mode does not isolate objects by logical bucket", bucket)
}
_, err := o.client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)})
if isS3NotFound(err) {
return nil
}
return err
}
// Copy copies an object from source to destination
func (o *OSSStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
srcBucket, srcPath = o.resolveBucketAndPath(srcBucket, srcPath)
+44 -1
View File
@@ -262,6 +262,9 @@ func (s *S3Storage) Remove(ctx context.Context, bucket, fnm string, tenantID ...
Key: aws.String(fnm),
})
if err != nil {
if isS3NotFound(err) {
return nil
}
common.Error("Failed to remove object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
return err
}
@@ -287,6 +290,18 @@ func (s *S3Storage) ObjExist(ctx context.Context, bucket, fnm string, tenantID .
return true
}
func (s *S3Storage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
_, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(fnm)})
if err == nil {
return true, nil
}
if isS3NotFound(err) {
return false, nil
}
return false, err
}
func (s *S3Storage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
listInput := &s3.ListObjectsV2Input{
@@ -349,6 +364,11 @@ func (s *S3Storage) BucketExists(ctx context.Context, bucket string) bool {
return true
}
func (s *S3Storage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
actualBucket, _ := s.resolveBucketAndPrefix(bucket)
return s.bucketExistsForRemoval(ctx, actualBucket)
}
// RemoveBucket removes a bucket and all its objects
func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {
actualBucket, prefix := s.resolveBucketAndPrefix(bucket)
@@ -378,6 +398,29 @@ func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {
return nil
}
// RemoveEmptyBucket removes a bucket only when it contains no object versions.
func (s *S3Storage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
actualBucket, prefix := s.resolveBucketAndPrefix(bucket)
exists, err := s.bucketExistsForRemoval(ctx, actualBucket)
if err != nil || !exists {
return err
}
if s.bucket != "" {
versions, err := s.client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
Bucket: aws.String(actualBucket), Prefix: aws.String(prefix), MaxKeys: aws.Int32(1),
})
if err != nil {
return err
}
if len(versions.Versions) > 0 || len(versions.DeleteMarkers) > 0 || (versions.IsTruncated != nil && *versions.IsTruncated) {
return fmt.Errorf("bucket %s is not empty", bucket)
}
return nil
}
_, err = s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(actualBucket)})
return err
}
func (s *S3Storage) resolveBucketAndPrefix(bucket string) (string, string) {
if s.bucket == "" {
return bucket, ""
@@ -490,7 +533,7 @@ func isS3NotFound(err error) bool {
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchKey"
return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchKey" || apiErr.ErrorCode() == "NoSuchBucket"
}
return false
}
+87
View File
@@ -129,6 +129,93 @@ func TestS3StorageRemoveBucketDeletesEmptyPhysicalBucket(t *testing.T) {
}
}
func TestS3StorageRemoveEmptyBucketPreservesNonemptyPhysicalBucket(t *testing.T) {
storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
switch req.Method {
case http.MethodHead:
return s3Response(http.StatusOK, ""), nil
case http.MethodDelete:
return s3Response(http.StatusConflict, `<Error><Code>BucketNotEmpty</Code></Error>`), nil
}
t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
return nil, nil
})
if err := storage.RemoveEmptyBucket(t.Context(), "kb01"); err == nil {
t.Fatal("RemoveEmptyBucket returned nil for a nonempty bucket")
}
}
func TestS3StorageExistenceChecksDistinguishMissingFromFailure(t *testing.T) {
for _, test := range []struct {
name string
status int
code string
missing bool
}{
{name: "missing", status: http.StatusNotFound, code: "NoSuchBucket", missing: true},
{name: "unavailable", status: http.StatusInternalServerError, code: "InternalError"},
} {
t.Run(test.name, func(t *testing.T) {
storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
return s3Response(test.status, `<Error><Code>`+test.code+`</Code></Error>`), nil
})
objectExists, objectErr := storage.ObjectExists(t.Context(), "kb01", "file")
bucketExists, bucketErr := storage.BucketExistsWithError(t.Context(), "kb01")
if objectExists || bucketExists {
t.Fatal("existence check reported an existing item")
}
if test.missing && (objectErr != nil || bucketErr != nil) {
t.Fatalf("missing item returned errors: object=%v bucket=%v", objectErr, bucketErr)
}
if !test.missing && (objectErr == nil || bucketErr == nil) {
t.Fatalf("storage failure was treated as missing: object=%v bucket=%v", objectErr, bucketErr)
}
})
}
}
func TestS3StorageRemoveEmptyBucketDeletesEmptyPhysicalBucket(t *testing.T) {
deleted := false
storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
switch req.Method {
case http.MethodHead:
return s3Response(http.StatusOK, ""), nil
case http.MethodDelete:
deleted = true
return s3Response(http.StatusNoContent, ""), nil
}
t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
return nil, nil
})
if err := storage.RemoveEmptyBucket(t.Context(), "kb01"); err != nil {
t.Fatal(err)
}
if !deleted {
t.Fatal("RemoveEmptyBucket did not delete an empty bucket")
}
}
func TestS3StorageRemoveEmptyBucketPreservesLogicalBucketObjects(t *testing.T) {
storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
switch {
case req.Method == http.MethodHead:
return s3Response(http.StatusOK, ""), nil
case req.URL.Query().Has("versions"):
if got := req.URL.Query().Get("prefix"); got != "prefix/kb01/" {
t.Fatalf("prefix = %q", got)
}
return s3Response(http.StatusOK, `<ListVersionsResult><Version><Key>prefix/kb01/file</Key><VersionId>v1</VersionId></Version></ListVersionsResult>`), nil
}
t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
return nil, nil
})
storage.bucket = "physical"
storage.prefixPath = "prefix"
if err := storage.RemoveEmptyBucket(t.Context(), "kb01"); err == nil {
t.Fatal("RemoveEmptyBucket returned nil for a nonempty logical bucket")
}
}
func TestS3StorageRemoveBucketSingleBucketMode(t *testing.T) {
var deleteBody string
storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
+7
View File
@@ -78,6 +78,8 @@ type Storage interface {
// ObjExist checks if an object exists
ObjExist(ctx context.Context, bucket, fnm string, tenantID ...string) bool
// ObjectExists distinguishes a missing object from a storage error
ObjectExists(ctx context.Context, bucket, fnm string) (bool, error)
// ListObjects list all objects of the bucket
ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error)
@@ -88,10 +90,15 @@ type Storage interface {
// BucketExists checks if a bucket exists
BucketExists(ctx context.Context, bucket string) bool
// BucketExistsWithError distinguishes a missing bucket from a storage error
BucketExistsWithError(ctx context.Context, bucket string) (bool, error)
// RemoveBucket removes a bucket and all its objects
RemoveBucket(ctx context.Context, bucket string) error
// RemoveEmptyBucket removes a bucket only if it has no objects
RemoveEmptyBucket(ctx context.Context, bucket string) error
// Copy copies an object from source to destination
Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool