mirror of
https://github.com/langgenius/dify-cloud-kit.git
synced 2026-07-18 18:44:27 -04:00
add azure blob implement
This commit is contained in:
Byron Wang
@@ -3,6 +3,7 @@ module github.com/langgenius/dify-cloud-kit
|
||||
go 1.23.3
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.14
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.67
|
||||
@@ -11,6 +12,8 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
|
||||
@@ -27,5 +30,7 @@ require (
|
||||
github.com/aws/smithy-go v1.22.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
golang.org/x/net v0.39.0 // indirect
|
||||
golang.org/x/text v0.24.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -1,3 +1,15 @@
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
|
||||
@@ -36,11 +48,34 @@ github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
|
||||
github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
|
||||
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
|
||||
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
|
||||
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
|
||||
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
|
||||
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
|
||||
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
)
|
||||
|
||||
type AzureBlobStorage struct {
|
||||
client *azblob.Client
|
||||
containerName string
|
||||
}
|
||||
|
||||
func NewAzureBlobStorage(args oss.OSSArgs) (oss.OSS, error) {
|
||||
connectionString := args.AzureBlob.ConnectionString
|
||||
containerName := args.AzureBlob.ContainerName
|
||||
client, err := azblob.NewClientFromConnectionString(connectionString, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &AzureBlobStorage{
|
||||
client: client,
|
||||
containerName: containerName,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Save(key string, data []byte) error {
|
||||
_, err := a.client.UploadBuffer(context.TODO(), a.containerName, key, data, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Load(key string) ([]byte, error) {
|
||||
get, err := a.client.DownloadStream(context.TODO(), a.containerName, key, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
downloadedData := bytes.Buffer{}
|
||||
retryReader := get.NewRetryReader(context.TODO(), &azblob.RetryReaderOptions{})
|
||||
_, err = downloadedData.ReadFrom(retryReader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = retryReader.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return downloadedData.Bytes(), nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Exists(key string) (bool, error) {
|
||||
blobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlobClient(key)
|
||||
_, err := blobClient.GetProperties(context.TODO(), nil)
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "404") {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) State(key string) (oss.OSSState, error) {
|
||||
blobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlobClient(key)
|
||||
props, err := blobClient.GetProperties(context.TODO(), nil)
|
||||
|
||||
if err != nil {
|
||||
return oss.OSSState{}, err
|
||||
}
|
||||
|
||||
return oss.OSSState{
|
||||
Size: *props.ContentLength,
|
||||
LastModified: *props.LastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) List(prefix string) ([]oss.OSSPath, error) {
|
||||
// append a slash to the prefix if it doesn't end with one
|
||||
if !strings.HasSuffix(prefix, "/") {
|
||||
prefix = prefix + "/"
|
||||
}
|
||||
|
||||
pager := a.client.NewListBlobsFlatPager(a.containerName, &azblob.ListBlobsFlatOptions{
|
||||
Prefix: &prefix,
|
||||
})
|
||||
|
||||
paths := make([]oss.OSSPath, 0)
|
||||
for pager.More() {
|
||||
page, err := pager.NextPage(context.TODO())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, blob := range page.Segment.BlobItems {
|
||||
// remove prefix
|
||||
key := strings.TrimPrefix(*blob.Name, prefix)
|
||||
// remove leading slash
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
paths = append(paths, oss.OSSPath{
|
||||
Path: key,
|
||||
IsDir: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Delete(key string) error {
|
||||
_, err := a.client.DeleteBlob(context.TODO(), a.containerName, key, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Type() string {
|
||||
return oss.OSS_TYPE_AZURE_BLOB
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package factory
|
||||
|
||||
import (
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/azureblob"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/local"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/s3"
|
||||
)
|
||||
@@ -11,6 +12,8 @@ var OSSFactory = map[string]func(oss.OSSArgs) (oss.OSS, error){
|
||||
"local_file": local.NewLocalStorage,
|
||||
"s3": s3.NewS3Storage,
|
||||
"aws_s3": s3.NewS3Storage,
|
||||
"azure": azureblob.NewAzureBlobStorage,
|
||||
"azure_blob": azureblob.NewAzureBlobStorage,
|
||||
}
|
||||
|
||||
func Load(name string, args oss.OSSArgs) (oss.OSS, error) {
|
||||
|
||||
@@ -59,6 +59,8 @@ type S3 struct {
|
||||
}
|
||||
|
||||
type AzureBlob struct {
|
||||
ConnectionString string
|
||||
ContainerName string
|
||||
}
|
||||
|
||||
type Local struct {
|
||||
|
||||
@@ -43,6 +43,15 @@ var allCases = []testArgsCases{
|
||||
},
|
||||
skip: true,
|
||||
},
|
||||
{
|
||||
vendor: "azure",
|
||||
args: oss.OSSArgs{
|
||||
AzureBlob: &oss.AzureBlob{
|
||||
ConnectionString: os.Getenv("AZURE_CONNECTION"),
|
||||
ContainerName: os.Getenv("AZURE_CONTAINER"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
|
||||
Reference in New Issue
Block a user