diff --git a/go.mod b/go.mod index 33ab4dc..cd7f945 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.23.3 require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.29.14 github.com/aws/aws-sdk-go-v2/credentials v1.17.67 @@ -32,5 +33,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/net v0.39.0 // indirect golang.org/x/text v0.24.0 // indirect + golang.org/x/time v0.11.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index cc87bbd..71ea96b 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSy github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= @@ -74,6 +76,8 @@ golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/oss/aliyun/aliyun.go b/oss/aliyun/aliyun.go new file mode 100644 index 0000000..5a5931a --- /dev/null +++ b/oss/aliyun/aliyun.go @@ -0,0 +1,194 @@ +package aliyun + +import ( + "bytes" + "fmt" + "io" + "path" + "strings" + "time" + + "github.com/aliyun/aliyun-oss-go-sdk/oss" + dify_oss "github.com/langgenius/dify-cloud-kit/oss" +) + +type AliyunOSSStorage struct { + client *oss.Client + bucket *oss.Bucket + path string +} + +func NewAliyunOSSStorage(args dify_oss.OSSArgs) (dify_oss.OSS, error) { + region := args.AliyunOSS.Region + endpoint := args.AliyunOSS.Endpoint + accessKeyID := args.AliyunOSS.AccessKey + accessKeySecret := args.AliyunOSS.SecretKey + authVersion := args.AliyunOSS.AuthVersion + path := args.AliyunOSS.Path + bucketName := args.AliyunOSS.Bucket + + // options + var options []oss.ClientOption + + // set region (required for v4) + if region != "" { + options = append(options, oss.Region(region)) + } + + // set auth-version + if authVersion == "v1" { + options = append(options, oss.AuthVersion(oss.AuthV1)) + } else if authVersion == "v4" { + options = append(options, oss.AuthVersion(oss.AuthV4)) + } else { + // default use v4 + options = append(options, oss.AuthVersion(oss.AuthV4)) + } + + // create client + var client *oss.Client + var err error + + client, err = oss.New(endpoint, accessKeyID, accessKeySecret, options...) + + if err != nil { + return nil, fmt.Errorf("failed to create AliyunOSS client: %w", err) + } + + // get specified bucket + bucket, err := client.Bucket(bucketName) + if err != nil { + return nil, fmt.Errorf("failed to get bucket %s: %w", bucketName, err) + } + + // normalize path: remove leading slash, ensure trailing slash + path = strings.TrimPrefix(path, "/") + if path != "" && !strings.HasSuffix(path, "/") { + path = path + "/" + } + + return &AliyunOSSStorage{ + client: client, + bucket: bucket, + path: path, + }, nil +} + +// combine full object path +func (s *AliyunOSSStorage) fullPath(key string) string { + return path.Join(s.path, key) +} + +func (s *AliyunOSSStorage) Save(key string, data []byte) error { + fullPath := s.fullPath(key) + return s.bucket.PutObject(fullPath, bytes.NewReader(data)) +} + +func (s *AliyunOSSStorage) Load(key string) ([]byte, error) { + fullPath := s.fullPath(key) + object, err := s.bucket.GetObject(fullPath) + if err != nil { + return nil, err + } + // Ensure object is closed after reading + defer object.Close() + + data, err := io.ReadAll(object) + if err != nil { + return nil, err + } + return data, nil +} + +func (s *AliyunOSSStorage) Exists(key string) (bool, error) { + fullPath := s.fullPath(key) + return s.bucket.IsObjectExist(fullPath) +} + +func (s *AliyunOSSStorage) State(key string) (dify_oss.OSSState, error) { + fullPath := s.fullPath(key) + meta, err := s.bucket.GetObjectMeta(fullPath) + if err != nil { + return dify_oss.OSSState{}, err + } + + // Get content length + size := int64(0) + contentLength := meta.Get("Content-Length") + if contentLength != "" { + _, err := fmt.Sscanf(contentLength, "%d", &size) + if err != nil { + // Return zero size if parsing fails + size = 0 + } + } + + // Get last modified time + lastModified := time.Time{} + lastModifiedStr := meta.Get("Last-Modified") + if lastModifiedStr != "" { + lastModified, err = time.Parse(time.RFC1123, lastModifiedStr) + if err != nil { + // Return zero time if parsing fails + lastModified = time.Time{} + } + } + + return dify_oss.OSSState{ + Size: size, + LastModified: lastModified, + }, nil +} + +func (s *AliyunOSSStorage) List(prefix string) ([]dify_oss.OSSPath, error) { + // combine given prefix with path + fullPrefix := s.fullPath(prefix) + + // Ensure the prefix ends with a slash for directories + if !strings.HasSuffix(fullPrefix, "/") { + fullPrefix = fullPrefix + "/" + } + + var keys []dify_oss.OSSPath + marker := "" + for { + lsRes, err := s.bucket.ListObjects(oss.Marker(marker), oss.Prefix(fullPrefix)) + if err != nil { + return nil, fmt.Errorf("failed to list objects in Aliyun OSS: %w", err) + } + + for _, object := range lsRes.Objects { + if object.Key == fullPrefix { + continue + } + // remove path and prefix from full path, only keep relative path + key := strings.TrimPrefix(object.Key, fullPrefix) + // Skip empty keys and directories (keys ending with /) + if key == "" || strings.HasSuffix(key, "/") { + continue + } + keys = append(keys, dify_oss.OSSPath{ + Path: key, + IsDir: false, + }) + } + + // Check if there are more results + if lsRes.IsTruncated { + marker = lsRes.NextMarker + } else { + break + } + } + + return keys, nil +} + +func (s *AliyunOSSStorage) Delete(key string) error { + fullPath := s.fullPath(key) + return s.bucket.DeleteObject(fullPath) +} + +func (s *AliyunOSSStorage) Type() string { + return dify_oss.OSS_TYPE_ALIYUN_OSS +} diff --git a/oss/factory/factory.go b/oss/factory/factory.go index 9f27c71..a2c105e 100644 --- a/oss/factory/factory.go +++ b/oss/factory/factory.go @@ -2,6 +2,7 @@ package factory import ( "github.com/langgenius/dify-cloud-kit/oss" + "github.com/langgenius/dify-cloud-kit/oss/aliyun" "github.com/langgenius/dify-cloud-kit/oss/azureblob" "github.com/langgenius/dify-cloud-kit/oss/local" "github.com/langgenius/dify-cloud-kit/oss/s3" @@ -14,6 +15,9 @@ var OSSFactory = map[string]func(oss.OSSArgs) (oss.OSS, error){ "aws_s3": s3.NewS3Storage, "azure": azureblob.NewAzureBlobStorage, "azure_blob": azureblob.NewAzureBlobStorage, + "aliyun": aliyun.NewAliyunOSSStorage, + "aliyun-oss": aliyun.NewAliyunOSSStorage, + "aliyun_oss": aliyun.NewAliyunOSSStorage, } func Load(name string, args oss.OSSArgs) (oss.OSS, error) { diff --git a/oss/oss.go b/oss/oss.go index 8e0ec4f..63f5fc0 100644 --- a/oss/oss.go +++ b/oss/oss.go @@ -40,12 +40,15 @@ type OSS interface { // Type returns the type of the storage // For example: local, aws_s3, tencent_cos Type() string + + //Validate() error } type OSSArgs struct { S3 *S3 Local *Local AzureBlob *AzureBlob + AliyunOSS *AliyunOSS } type S3 struct { @@ -66,3 +69,13 @@ type AzureBlob struct { type Local struct { Path string } + +type AliyunOSS struct { + Region string + Endpoint string + AccessKey string + SecretKey string + AuthVersion string + Path string + Bucket string +} diff --git a/tests/oss/oss_test.go b/tests/oss/oss_test.go index 069637e..26aca38 100644 --- a/tests/oss/oss_test.go +++ b/tests/oss/oss_test.go @@ -51,6 +51,21 @@ var allCases = []testArgsCases{ ContainerName: os.Getenv("AZURE_CONTAINER"), }, }, + skip: true, + }, + { + vendor: "aliyun", + args: oss.OSSArgs{ + AliyunOSS: &oss.AliyunOSS{ + Region: os.Getenv("ALIYUN_OSS_REGION"), + Endpoint: os.Getenv("ALIYUN_OSS_ENDPOINT"), + AccessKey: os.Getenv("ALIYUN_OSS_ACCESS_KEY"), + SecretKey: os.Getenv("ALIYUN_OSS_SECRET_KEY"), + AuthVersion: os.Getenv("ALIYUN_OSS_AUTH_VERSION"), + Path: os.Getenv("ALIYUN_OSS_PATH"), + Bucket: os.Getenv("ALIYUN_OSS_BUCKET"), + }, + }, }, }