add aliyun oss implement

2026-07-18 18:44:27 -04:00 · 2025-05-27 12:29:12 +08:00
parent 0167ff48e4
commit a723351e0b
6 changed files with 232 additions and 0 deletions
@@ -4,6 +4,7 @@ go 1.23.3

 require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
+	github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
 	github.com/aws/aws-sdk-go-v2 v1.36.3
 	github.com/aws/aws-sdk-go-v2/config v1.29.14
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.67
@@ -32,5 +33,6 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/net v0.39.0 // indirect
 	golang.org/x/text v0.24.0 // indirect
+	golang.org/x/time v0.11.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
@@ -10,6 +10,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSy
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
+github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
 github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
@@ -74,6 +76,8 @@ golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
 golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
 golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
+golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
+golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@@ -0,0 +1,194 @@
+package aliyun
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	dify_oss "github.com/langgenius/dify-cloud-kit/oss"
+)
+
+type AliyunOSSStorage struct {
+	client *oss.Client
+	bucket *oss.Bucket
+	path   string
+}
+
+func NewAliyunOSSStorage(args dify_oss.OSSArgs) (dify_oss.OSS, error) {
+	region := args.AliyunOSS.Region
+	endpoint := args.AliyunOSS.Endpoint
+	accessKeyID := args.AliyunOSS.AccessKey
+	accessKeySecret := args.AliyunOSS.SecretKey
+	authVersion := args.AliyunOSS.AuthVersion
+	path := args.AliyunOSS.Path
+	bucketName := args.AliyunOSS.Bucket
+
+	// options
+	var options []oss.ClientOption
+
+	// set region (required for v4)
+	if region != "" {
+		options = append(options, oss.Region(region))
+	}
+
+	// set auth-version
+	if authVersion == "v1" {
+		options = append(options, oss.AuthVersion(oss.AuthV1))
+	} else if authVersion == "v4" {
+		options = append(options, oss.AuthVersion(oss.AuthV4))
+	} else {
+		// default use v4
+		options = append(options, oss.AuthVersion(oss.AuthV4))
+	}
+
+	// create client
+	var client *oss.Client
+	var err error
+
+	client, err = oss.New(endpoint, accessKeyID, accessKeySecret, options...)
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to create AliyunOSS client: %w", err)
+	}
+
+	// get specified bucket
+	bucket, err := client.Bucket(bucketName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get bucket %s: %w", bucketName, err)
+	}
+
+	// normalize path: remove leading slash, ensure trailing slash
+	path = strings.TrimPrefix(path, "/")
+	if path != "" && !strings.HasSuffix(path, "/") {
+		path = path + "/"
+	}
+
+	return &AliyunOSSStorage{
+		client: client,
+		bucket: bucket,
+		path:   path,
+	}, nil
+}
+
+// combine full object path
+func (s *AliyunOSSStorage) fullPath(key string) string {
+	return path.Join(s.path, key)
+}
+
+func (s *AliyunOSSStorage) Save(key string, data []byte) error {
+	fullPath := s.fullPath(key)
+	return s.bucket.PutObject(fullPath, bytes.NewReader(data))
+}
+
+func (s *AliyunOSSStorage) Load(key string) ([]byte, error) {
+	fullPath := s.fullPath(key)
+	object, err := s.bucket.GetObject(fullPath)
+	if err != nil {
+		return nil, err
+	}
+	// Ensure object is closed after reading
+	defer object.Close()
+
+	data, err := io.ReadAll(object)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func (s *AliyunOSSStorage) Exists(key string) (bool, error) {
+	fullPath := s.fullPath(key)
+	return s.bucket.IsObjectExist(fullPath)
+}
+
+func (s *AliyunOSSStorage) State(key string) (dify_oss.OSSState, error) {
+	fullPath := s.fullPath(key)
+	meta, err := s.bucket.GetObjectMeta(fullPath)
+	if err != nil {
+		return dify_oss.OSSState{}, err
+	}
+
+	// Get content length
+	size := int64(0)
+	contentLength := meta.Get("Content-Length")
+	if contentLength != "" {
+		_, err := fmt.Sscanf(contentLength, "%d", &size)
+		if err != nil {
+			// Return zero size if parsing fails
+			size = 0
+		}
+	}
+
+	// Get last modified time
+	lastModified := time.Time{}
+	lastModifiedStr := meta.Get("Last-Modified")
+	if lastModifiedStr != "" {
+		lastModified, err = time.Parse(time.RFC1123, lastModifiedStr)
+		if err != nil {
+			// Return zero time if parsing fails
+			lastModified = time.Time{}
+		}
+	}
+
+	return dify_oss.OSSState{
+		Size:         size,
+		LastModified: lastModified,
+	}, nil
+}
+
+func (s *AliyunOSSStorage) List(prefix string) ([]dify_oss.OSSPath, error) {
+	// combine given prefix with path
+	fullPrefix := s.fullPath(prefix)
+
+	// Ensure the prefix ends with a slash for directories
+	if !strings.HasSuffix(fullPrefix, "/") {
+		fullPrefix = fullPrefix + "/"
+	}
+
+	var keys []dify_oss.OSSPath
+	marker := ""
+	for {
+		lsRes, err := s.bucket.ListObjects(oss.Marker(marker), oss.Prefix(fullPrefix))
+		if err != nil {
+			return nil, fmt.Errorf("failed to list objects in Aliyun OSS: %w", err)
+		}
+
+		for _, object := range lsRes.Objects {
+			if object.Key == fullPrefix {
+				continue
+			}
+			// remove path and prefix from full path, only keep relative path
+			key := strings.TrimPrefix(object.Key, fullPrefix)
+			// Skip empty keys and directories (keys ending with /)
+			if key == "" || strings.HasSuffix(key, "/") {
+				continue
+			}
+			keys = append(keys, dify_oss.OSSPath{
+				Path:  key,
+				IsDir: false,
+			})
+		}
+
+		// Check if there are more results
+		if lsRes.IsTruncated {
+			marker = lsRes.NextMarker
+		} else {
+			break
+		}
+	}
+
+	return keys, nil
+}
+
+func (s *AliyunOSSStorage) Delete(key string) error {
+	fullPath := s.fullPath(key)
+	return s.bucket.DeleteObject(fullPath)
+}
+
+func (s *AliyunOSSStorage) Type() string {
+	return dify_oss.OSS_TYPE_ALIYUN_OSS
+}
@@ -2,6 +2,7 @@ package factory

 import (
 	"github.com/langgenius/dify-cloud-kit/oss"
+	"github.com/langgenius/dify-cloud-kit/oss/aliyun"
 	"github.com/langgenius/dify-cloud-kit/oss/azureblob"
 	"github.com/langgenius/dify-cloud-kit/oss/local"
 	"github.com/langgenius/dify-cloud-kit/oss/s3"
@@ -14,6 +15,9 @@ var OSSFactory = map[string]func(oss.OSSArgs) (oss.OSS, error){
 	"aws_s3":     s3.NewS3Storage,
 	"azure":      azureblob.NewAzureBlobStorage,
 	"azure_blob": azureblob.NewAzureBlobStorage,
+	"aliyun":     aliyun.NewAliyunOSSStorage,
+	"aliyun-oss": aliyun.NewAliyunOSSStorage,
+	"aliyun_oss": aliyun.NewAliyunOSSStorage,
 }

 func Load(name string, args oss.OSSArgs) (oss.OSS, error) {
@@ -40,12 +40,15 @@ type OSS interface {
 	// Type returns the type of the storage
 	// For example: local, aws_s3, tencent_cos
 	Type() string
+
+	//Validate() error
 }

 type OSSArgs struct {
 	S3        *S3
 	Local     *Local
 	AzureBlob *AzureBlob
+	AliyunOSS *AliyunOSS
 }

 type S3 struct {
@@ -66,3 +69,13 @@ type AzureBlob struct {
 type Local struct {
 	Path string
 }
+
+type AliyunOSS struct {
+	Region      string
+	Endpoint    string
+	AccessKey   string
+	SecretKey   string
+	AuthVersion string
+	Path        string
+	Bucket      string
+}
@@ -51,6 +51,21 @@ var allCases = []testArgsCases{
 				ContainerName:    os.Getenv("AZURE_CONTAINER"),
 			},
 		},
+		skip: true,
+	},
+	{
+		vendor: "aliyun",
+		args: oss.OSSArgs{
+			AliyunOSS: &oss.AliyunOSS{
+				Region:      os.Getenv("ALIYUN_OSS_REGION"),
+				Endpoint:    os.Getenv("ALIYUN_OSS_ENDPOINT"),
+				AccessKey:   os.Getenv("ALIYUN_OSS_ACCESS_KEY"),
+				SecretKey:   os.Getenv("ALIYUN_OSS_SECRET_KEY"),
+				AuthVersion: os.Getenv("ALIYUN_OSS_AUTH_VERSION"),
+				Path:        os.Getenv("ALIYUN_OSS_PATH"),
+				Bucket:      os.Getenv("ALIYUN_OSS_BUCKET"),
+			},
+		},
 	},
 }