mirror of
https://github.com/langgenius/dify-cloud-kit.git
synced 2026-07-25 21:45:26 -04:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f932ae4155 | |||
| bdc2eef3cd | |||
| b5d897642a | |||
| d56e719c49 | |||
| c4a09cd10a | |||
| f560f324a2 | |||
| dd700497ef |
@@ -48,7 +48,7 @@ func main() {
|
||||
| Azure Blob | `oss/azureblob/blob.go` | `AccountName`, `AccountKey`, `Container` |
|
||||
| Google GCS | `oss/gcsblob/gcs.go` | `CredentialsJSON`, `Bucket` |
|
||||
| Tencent COS | `oss/tencentcos/cos.go` | `SecretId`, `SecretKey`, `Bucket`, `Region` |
|
||||
| Huawei OBS | `oss/huanweiobs/obs.go` | `AK`, `SK`, `Endpoint`, `Bucket` |
|
||||
| Huawei OBS | `oss/huaweiobs/obs.go` | `AK`, `SK`, `Endpoint`, `Bucket` |
|
||||
| Volcengine TOS | `oss/volcenginetos/tos.go` | `Endpoint`, `AccessKey`, `SecretKey`, `Bucket` |
|
||||
|
||||
## 🏗️ Usage with Factory
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
module github.com/langgenius/dify-cloud-kit
|
||||
|
||||
go 1.23.0
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go/storage v1.54.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3
|
||||
@@ -28,6 +29,7 @@ require (
|
||||
cloud.google.com/go/monitoring v1.24.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
|
||||
@@ -51,16 +53,19 @@ require (
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.0.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
|
||||
github.com/google/go-querystring v1.0.0 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.14.1 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.3 // indirect
|
||||
github.com/mozillazg/go-httpheader v0.2.1 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
|
||||
@@ -74,12 +79,12 @@ require (
|
||||
go.opentelemetry.io/otel/sdk v1.35.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.35.0 // indirect
|
||||
golang.org/x/crypto v0.37.0 // indirect
|
||||
golang.org/x/net v0.39.0 // indirect
|
||||
golang.org/x/crypto v0.45.0 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/oauth2 v0.30.0 // indirect
|
||||
golang.org/x/sync v0.14.0 // indirect
|
||||
golang.org/x/sys v0.32.0 // indirect
|
||||
golang.org/x/text v0.24.0 // indirect
|
||||
golang.org/x/sync v0.18.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
golang.org/x/time v0.11.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect
|
||||
|
||||
@@ -24,12 +24,16 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc=
|
||||
@@ -89,6 +93,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M=
|
||||
github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A=
|
||||
@@ -99,8 +105,8 @@ github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfU
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E=
|
||||
github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc=
|
||||
github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE=
|
||||
github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
||||
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
@@ -128,6 +134,8 @@ github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrk
|
||||
github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA=
|
||||
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible h1:yNjwdvn9fwuN6Ouxr0xHM0cVu03YMUWUyFmu2van/Yc=
|
||||
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
|
||||
github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
|
||||
github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
@@ -147,6 +155,8 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM=
|
||||
github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA=
|
||||
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
|
||||
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
@@ -188,21 +198,22 @@ go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5J
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
|
||||
go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
|
||||
go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
|
||||
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
|
||||
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
|
||||
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
|
||||
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
|
||||
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
|
||||
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
|
||||
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
|
||||
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
|
||||
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
|
||||
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
|
||||
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
|
||||
golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
||||
+21
-6
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
)
|
||||
@@ -15,20 +16,34 @@ type AzureBlobStorage struct {
|
||||
}
|
||||
|
||||
func NewAzureBlobStorage(args oss.OSSArgs) (oss.OSS, error) {
|
||||
var client *azblob.Client
|
||||
var err error
|
||||
|
||||
if args.AzureBlob == nil {
|
||||
return nil, oss.ErrArgumentInvalid.WithDetail("can't find Azure Blob argument in OSSArgs")
|
||||
}
|
||||
err := args.AzureBlob.Validate()
|
||||
err = args.AzureBlob.Validate()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
connectionString := args.AzureBlob.ConnectionString
|
||||
containerName := args.AzureBlob.ContainerName
|
||||
client, err := azblob.NewClientFromConnectionString(connectionString, nil)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
if args.AzureBlob.UseManagedIdentity {
|
||||
cred, err := azidentity.NewDefaultAzureCredential(nil)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
client, err = azblob.NewClient(args.AzureBlob.ServiceURL, cred, nil)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
} else {
|
||||
connectionString := args.AzureBlob.ConnectionString
|
||||
client, err = azblob.NewClientFromConnectionString(connectionString, nil)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
|
||||
}
|
||||
return &AzureBlobStorage{
|
||||
client: client,
|
||||
containerName: containerName,
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/langgenius/dify-cloud-kit/oss/aliyun"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/azureblob"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/gcsblob"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/huanweiobs"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/huaweiobs"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/local"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/s3"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/tencentcos"
|
||||
@@ -38,9 +38,9 @@ var OSSFactory = map[string]func(oss.OSSArgs) (oss.OSS, error){
|
||||
"google-storage": gcsblob.NewGoogleCloudStorage,
|
||||
"google_storage": gcsblob.NewGoogleCloudStorage,
|
||||
|
||||
"huawei": huanweiobs.NewHuaweiOBSStorage,
|
||||
"huawei-obs": huanweiobs.NewHuaweiOBSStorage,
|
||||
"huawei_obs": huanweiobs.NewHuaweiOBSStorage,
|
||||
"huawei": huaweiobs.NewHuaweiOBSStorage,
|
||||
"huawei-obs": huaweiobs.NewHuaweiOBSStorage,
|
||||
"huawei_obs": huaweiobs.NewHuaweiOBSStorage,
|
||||
|
||||
"volcengine": volcenginetos.NewVolcengineTOSStorage,
|
||||
"volcengine_tos": volcenginetos.NewVolcengineTOSStorage,
|
||||
|
||||
+17
-8
@@ -29,13 +29,22 @@ func NewGoogleCloudStorage(args oss.OSSArgs) (oss.OSS, error) {
|
||||
ctx := context.Background()
|
||||
bucket := args.GoogleCloudStorage.Bucket
|
||||
credentialsB64 := args.GoogleCloudStorage.CredentialsB64
|
||||
credentials, err := base64.StdEncoding.DecodeString(credentialsB64)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err).WithDetail("credentials must be a base64 encoded string")
|
||||
}
|
||||
client, err := storage.NewClient(ctx, option.WithCredentialsJSON(credentials))
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
|
||||
var client *storage.Client
|
||||
if credentialsB64 == "" {
|
||||
client, err = storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err).WithDetail("failed to initialize GCS client with ADC")
|
||||
}
|
||||
} else {
|
||||
credentials, err := base64.StdEncoding.DecodeString(credentialsB64)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err).WithDetail("credentials must be a base64 encoded string")
|
||||
}
|
||||
client, err = storage.NewClient(ctx, option.WithCredentialsJSON(credentials))
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
}
|
||||
return &GoogleCloudStorage{
|
||||
bucket: bucket,
|
||||
@@ -116,7 +125,7 @@ func (g *GoogleCloudStorage) List(prefix string) ([]oss.OSSPath, error) {
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
|
||||
res = append(res, oss.OSSPath{
|
||||
Path: attrs.Name,
|
||||
Path: key,
|
||||
IsDir: false,
|
||||
})
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package huanweiobs
|
||||
package huaweiobs
|
||||
|
||||
import (
|
||||
"io"
|
||||
@@ -115,30 +115,43 @@ func (h *HuaweiOBSStorage) State(key string) (oss.OSSState, error) {
|
||||
}
|
||||
|
||||
func (h *HuaweiOBSStorage) List(prefix string) ([]oss.OSSPath, error) {
|
||||
output, err := h.client.ListObjects(&obs.ListObjectsInput{
|
||||
Bucket: h.bucket,
|
||||
ListObjsInput: obs.ListObjsInput{
|
||||
Prefix: prefix,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if !strings.HasSuffix(prefix, "/") {
|
||||
prefix = prefix + "/"
|
||||
}
|
||||
paths := []oss.OSSPath{}
|
||||
for _, v := range output.Contents {
|
||||
key := strings.TrimPrefix(v.Key, prefix)
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
paths = append(paths, oss.OSSPath{
|
||||
Path: key,
|
||||
IsDir: func() bool {
|
||||
return strings.HasSuffix(v.Key, "/")
|
||||
}(),
|
||||
marker := ""
|
||||
paths := []oss.OSSPath{}
|
||||
for {
|
||||
output, err := h.client.ListObjects(&obs.ListObjectsInput{
|
||||
Bucket: h.bucket,
|
||||
ListObjsInput: obs.ListObjsInput{
|
||||
Prefix: prefix,
|
||||
},
|
||||
Marker: marker,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, v := range output.Contents {
|
||||
key := strings.TrimPrefix(v.Key, prefix)
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
paths = append(paths, oss.OSSPath{
|
||||
Path: key,
|
||||
IsDir: false,
|
||||
})
|
||||
}
|
||||
if output.IsTruncated {
|
||||
marker = output.NextMarker
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
+21
-7
@@ -72,22 +72,35 @@ type S3 struct {
|
||||
|
||||
func (s *S3) Validate() error {
|
||||
if s.Bucket == "" || s.Region == "" {
|
||||
msg := fmt.Sprintf("bucket and region cannot be empty.")
|
||||
msg := "bucket and region cannot be empty."
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type AzureBlob struct {
|
||||
ConnectionString string
|
||||
ContainerName string
|
||||
ConnectionString string
|
||||
ContainerName string
|
||||
UseManagedIdentity bool
|
||||
ServiceURL string
|
||||
}
|
||||
|
||||
func (a *AzureBlob) Validate() error {
|
||||
if a.ConnectionString == "" || a.ContainerName == "" {
|
||||
msg := fmt.Sprintf("connectorString and containerName cannot be empty.")
|
||||
if a.ContainerName == "" {
|
||||
msg := "containerName cannot be empty"
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
if a.UseManagedIdentity {
|
||||
if a.ServiceURL == "" {
|
||||
msg := "service url cannot be empty. e.g. https://<account>.blob.core.windows.net/."
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
} else {
|
||||
if a.ConnectionString == "" {
|
||||
msg := "connectorString cannot be empty."
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -126,6 +139,7 @@ type TencentCOS struct {
|
||||
SecretID string
|
||||
SecretKey string
|
||||
Bucket string
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
func (t *TencentCOS) Validate() error {
|
||||
@@ -142,8 +156,8 @@ type GoogleCloudStorage struct {
|
||||
}
|
||||
|
||||
func (g *GoogleCloudStorage) Validate() error {
|
||||
if g.Bucket == "" || g.CredentialsB64 == "" {
|
||||
msg := fmt.Sprintf("bucket and credentials cannot be empty.")
|
||||
if g.Bucket == "" {
|
||||
msg := fmt.Sprintf("bucket cannot be empty.")
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -33,7 +33,11 @@ func NewTencentCOSStorage(args oss.OSSArgs) (oss.OSS, error) {
|
||||
region := args.TencentCOS.Region
|
||||
secretID := args.TencentCOS.SecretID
|
||||
secretKey := args.TencentCOS.SecretKey
|
||||
u, err := url.Parse("https://" + bucket + ".cos." + region + ".myqcloud.com")
|
||||
endpoint := args.TencentCOS.Endpoint
|
||||
if endpoint == "" {
|
||||
endpoint = "https://" + bucket + ".cos." + region + ".myqcloud.com"
|
||||
}
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err).WithDetail("url parse failed")
|
||||
}
|
||||
|
||||
@@ -118,6 +118,17 @@ var allCases = []testArgsCases{
|
||||
},
|
||||
skip: false,
|
||||
},
|
||||
{
|
||||
vendor: "azure",
|
||||
args: oss.OSSArgs{
|
||||
AzureBlob: &oss.AzureBlob{
|
||||
UseManagedIdentity: true,
|
||||
ContainerName: os.Getenv("AZURE_CONTAINER"),
|
||||
ServiceURL: os.Getenv("AZURE_SERVICE_URL"),
|
||||
},
|
||||
},
|
||||
skip: true,
|
||||
},
|
||||
}
|
||||
|
||||
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
@@ -174,6 +185,9 @@ func TestAll(t *testing.T) {
|
||||
ossPaths, err = storage.List(prefix)
|
||||
assert.Equal(t, 1, len(ossPaths), info)
|
||||
assert.Nil(t, err, info)
|
||||
// Verify that List returns relative paths (without prefix)
|
||||
expectedRelativePath := key[len(prefix)+1:] // +1 for the "/" separator
|
||||
assert.Equal(t, expectedRelativePath, ossPaths[0].Path, info+" - List should return relative path")
|
||||
|
||||
err = storage.Delete(key)
|
||||
assert.Nil(t, err, info)
|
||||
|
||||
Reference in New Issue
Block a user