diff --git a/go.mod b/go.mod index 4ec0d35..33ab4dc 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/langgenius/dify-cloud-kit go 1.23.3 require ( + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.29.14 github.com/aws/aws-sdk-go-v2/credentials v1.17.67 @@ -11,6 +12,8 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect @@ -27,5 +30,7 @@ require ( github.com/aws/smithy-go v1.22.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/text v0.24.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index af7a7f0..cc87bbd 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,15 @@ +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= @@ -36,11 +48,34 @@ github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/oss/azureblob/blob.go b/oss/azureblob/blob.go new file mode 100644 index 0000000..fe1d5c9 --- /dev/null +++ b/oss/azureblob/blob.go @@ -0,0 +1,124 @@ +package azureblob + +import ( + "bytes" + "context" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/langgenius/dify-cloud-kit/oss" +) + +type AzureBlobStorage struct { + client *azblob.Client + containerName string +} + +func NewAzureBlobStorage(args oss.OSSArgs) (oss.OSS, error) { + connectionString := args.AzureBlob.ConnectionString + containerName := args.AzureBlob.ContainerName + client, err := azblob.NewClientFromConnectionString(connectionString, nil) + if err != nil { + return nil, err + } + + return &AzureBlobStorage{ + client: client, + containerName: containerName, + }, nil +} + +func (a *AzureBlobStorage) Save(key string, data []byte) error { + _, err := a.client.UploadBuffer(context.TODO(), a.containerName, key, data, nil) + return err +} + +func (a *AzureBlobStorage) Load(key string) ([]byte, error) { + get, err := a.client.DownloadStream(context.TODO(), a.containerName, key, nil) + if err != nil { + return nil, err + } + + downloadedData := bytes.Buffer{} + retryReader := get.NewRetryReader(context.TODO(), &azblob.RetryReaderOptions{}) + _, err = downloadedData.ReadFrom(retryReader) + if err != nil { + return nil, err + } + + err = retryReader.Close() + if err != nil { + return nil, err + } + + return downloadedData.Bytes(), nil +} + +func (a *AzureBlobStorage) Exists(key string) (bool, error) { + blobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlobClient(key) + _, err := blobClient.GetProperties(context.TODO(), nil) + + if err != nil { + if strings.Contains(err.Error(), "404") { + return false, nil + } + return false, err + } + + return true, nil +} + +func (a *AzureBlobStorage) State(key string) (oss.OSSState, error) { + blobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlobClient(key) + props, err := blobClient.GetProperties(context.TODO(), nil) + + if err != nil { + return oss.OSSState{}, err + } + + return oss.OSSState{ + Size: *props.ContentLength, + LastModified: *props.LastModified, + }, nil +} + +func (a *AzureBlobStorage) List(prefix string) ([]oss.OSSPath, error) { + // append a slash to the prefix if it doesn't end with one + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + + pager := a.client.NewListBlobsFlatPager(a.containerName, &azblob.ListBlobsFlatOptions{ + Prefix: &prefix, + }) + + paths := make([]oss.OSSPath, 0) + for pager.More() { + page, err := pager.NextPage(context.TODO()) + if err != nil { + return nil, err + } + + for _, blob := range page.Segment.BlobItems { + // remove prefix + key := strings.TrimPrefix(*blob.Name, prefix) + // remove leading slash + key = strings.TrimPrefix(key, "/") + paths = append(paths, oss.OSSPath{ + Path: key, + IsDir: false, + }) + } + } + + return paths, nil +} + +func (a *AzureBlobStorage) Delete(key string) error { + _, err := a.client.DeleteBlob(context.TODO(), a.containerName, key, nil) + return err +} + +func (a *AzureBlobStorage) Type() string { + return oss.OSS_TYPE_AZURE_BLOB +} diff --git a/oss/factory/factory.go b/oss/factory/factory.go index a8c05e2..9f27c71 100644 --- a/oss/factory/factory.go +++ b/oss/factory/factory.go @@ -2,6 +2,7 @@ package factory import ( "github.com/langgenius/dify-cloud-kit/oss" + "github.com/langgenius/dify-cloud-kit/oss/azureblob" "github.com/langgenius/dify-cloud-kit/oss/local" "github.com/langgenius/dify-cloud-kit/oss/s3" ) @@ -11,6 +12,8 @@ var OSSFactory = map[string]func(oss.OSSArgs) (oss.OSS, error){ "local_file": local.NewLocalStorage, "s3": s3.NewS3Storage, "aws_s3": s3.NewS3Storage, + "azure": azureblob.NewAzureBlobStorage, + "azure_blob": azureblob.NewAzureBlobStorage, } func Load(name string, args oss.OSSArgs) (oss.OSS, error) { diff --git a/oss/oss.go b/oss/oss.go index 8f524c4..8e0ec4f 100644 --- a/oss/oss.go +++ b/oss/oss.go @@ -59,6 +59,8 @@ type S3 struct { } type AzureBlob struct { + ConnectionString string + ContainerName string } type Local struct { diff --git a/tests/oss/oss_test.go b/tests/oss/oss_test.go index eab9a5e..069637e 100644 --- a/tests/oss/oss_test.go +++ b/tests/oss/oss_test.go @@ -43,6 +43,15 @@ var allCases = []testArgsCases{ }, skip: true, }, + { + vendor: "azure", + args: oss.OSSArgs{ + AzureBlob: &oss.AzureBlob{ + ConnectionString: os.Getenv("AZURE_CONNECTION"), + ContainerName: os.Getenv("AZURE_CONTAINER"), + }, + }, + }, } const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"