7 Commits

Author SHA1 Message Date
Byron.wang f932ae4155 add azure managed identity support (#24) 2026-04-07 09:45:17 +08:00
Daisuke Mino bdc2eef3cd fix(gcs): List returns relative path instead of full path (#20)
The GCS List method was returning full paths (attrs.Name) instead of
relative paths (key with prefix stripped). This was inconsistent with
other storage implementations (S3, Local, Azure, Aliyun, Tencent,
Huawei, Volcengine) which all return relative paths.

The original implementation in dify-plugin-daemon PR #237 correctly
used `key`:
https://github.com/langgenius/dify-plugin-daemon/pull/237/files#diff-1efde200d0fa3fafdd827478fdbca6e8b16dee52955cb3c93faab2849f7d95bfR136

This bug was introduced when the code was ported to dify-cloud-kit.

Also added a test assertion to verify that List returns relative paths.

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-01 20:41:37 +08:00
Minsoo Jun b5d897642a feat(gcs): add support for application default credentials (ADC) (#19)
Allows GCS initialization without explicit credentials to support ADC. Fixes #18.
2026-02-01 20:40:17 +08:00
dependabot[bot] d56e719c49 chore(deps): bump golang.org/x/crypto from 0.37.0 to 0.45.0 (#17)
Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.37.0 to 0.45.0.
- [Commits](https://github.com/golang/crypto/compare/v0.37.0...v0.45.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-version: 0.45.0
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-12-25 14:43:50 +08:00
sHuXnHs c4a09cd10a tx cos support endpoint (#13) 2025-11-18 11:11:49 +08:00
jzymx50 f560f324a2 fix(huawei obs): add folder suffix (#16) 2025-11-18 11:11:06 +08:00
dependabot[bot] dd700497ef chore(deps): bump github.com/go-jose/go-jose/v4 from 4.0.4 to 4.0.5 (#14)
Bumps [github.com/go-jose/go-jose/v4](https://github.com/go-jose/go-jose) from 4.0.4 to 4.0.5.
- [Release notes](https://github.com/go-jose/go-jose/releases)
- [Commits](https://github.com/go-jose/go-jose/compare/v4.0.4...v4.0.5)

---
updated-dependencies:
- dependency-name: github.com/go-jose/go-jose/v4
  dependency-version: 4.0.5
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-10-14 15:38:50 +08:00
10 changed files with 152 additions and 67 deletions
+1 -1
View File
@@ -48,7 +48,7 @@ func main() {
| Azure Blob | `oss/azureblob/blob.go` | `AccountName`, `AccountKey`, `Container` |
| Google GCS | `oss/gcsblob/gcs.go` | `CredentialsJSON`, `Bucket` |
| Tencent COS | `oss/tencentcos/cos.go` | `SecretId`, `SecretKey`, `Bucket`, `Region` |
| Huawei OBS | `oss/huanweiobs/obs.go` | `AK`, `SK`, `Endpoint`, `Bucket` |
| Huawei OBS | `oss/huaweiobs/obs.go` | `AK`, `SK`, `Endpoint`, `Bucket` |
| Volcengine TOS | `oss/volcenginetos/tos.go` | `Endpoint`, `AccessKey`, `SecretKey`, `Bucket` |
## 🏗️ Usage with Factory
+12 -7
View File
@@ -1,9 +1,10 @@
module github.com/langgenius/dify-cloud-kit
go 1.23.0
go 1.24.0
require (
cloud.google.com/go/storage v1.54.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
github.com/aws/aws-sdk-go-v2 v1.36.3
@@ -28,6 +29,7 @@ require (
cloud.google.com/go/monitoring v1.24.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
@@ -51,16 +53,19 @@ require (
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-jose/go-jose/v4 v4.0.4 // indirect
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
github.com/google/go-querystring v1.0.0 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
github.com/googleapis/gax-go/v2 v2.14.1 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/mozillazg/go-httpheader v0.2.1 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
@@ -74,12 +79,12 @@ require (
go.opentelemetry.io/otel/sdk v1.35.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
go.opentelemetry.io/otel/trace v1.35.0 // indirect
golang.org/x/crypto v0.37.0 // indirect
golang.org/x/net v0.39.0 // indirect
golang.org/x/crypto v0.45.0 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sync v0.14.0 // indirect
golang.org/x/sys v0.32.0 // indirect
golang.org/x/text v0.24.0 // indirect
golang.org/x/sync v0.18.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/text v0.31.0 // indirect
golang.org/x/time v0.11.0 // indirect
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect
+23 -12
View File
@@ -24,12 +24,16 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc=
@@ -89,6 +93,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M=
github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA=
github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A=
@@ -99,8 +105,8 @@ github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfU
github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E=
github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc=
github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE=
github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -128,6 +134,8 @@ github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrk
github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA=
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible h1:yNjwdvn9fwuN6Ouxr0xHM0cVu03YMUWUyFmu2van/Yc=
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -147,6 +155,8 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM=
github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
@@ -188,21 +198,22 @@ go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5J
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+21 -6
View File
@@ -5,6 +5,7 @@ import (
"context"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/langgenius/dify-cloud-kit/oss"
)
@@ -15,20 +16,34 @@ type AzureBlobStorage struct {
}
func NewAzureBlobStorage(args oss.OSSArgs) (oss.OSS, error) {
var client *azblob.Client
var err error
if args.AzureBlob == nil {
return nil, oss.ErrArgumentInvalid.WithDetail("can't find Azure Blob argument in OSSArgs")
}
err := args.AzureBlob.Validate()
err = args.AzureBlob.Validate()
if err != nil {
return nil, err
}
connectionString := args.AzureBlob.ConnectionString
containerName := args.AzureBlob.ContainerName
client, err := azblob.NewClientFromConnectionString(connectionString, nil)
if err != nil {
return nil, oss.ErrProviderInit.WithError(err)
}
if args.AzureBlob.UseManagedIdentity {
cred, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
return nil, oss.ErrProviderInit.WithError(err)
}
client, err = azblob.NewClient(args.AzureBlob.ServiceURL, cred, nil)
if err != nil {
return nil, oss.ErrProviderInit.WithError(err)
}
} else {
connectionString := args.AzureBlob.ConnectionString
client, err = azblob.NewClientFromConnectionString(connectionString, nil)
if err != nil {
return nil, oss.ErrProviderInit.WithError(err)
}
}
return &AzureBlobStorage{
client: client,
containerName: containerName,
+4 -4
View File
@@ -7,7 +7,7 @@ import (
"github.com/langgenius/dify-cloud-kit/oss/aliyun"
"github.com/langgenius/dify-cloud-kit/oss/azureblob"
"github.com/langgenius/dify-cloud-kit/oss/gcsblob"
"github.com/langgenius/dify-cloud-kit/oss/huanweiobs"
"github.com/langgenius/dify-cloud-kit/oss/huaweiobs"
"github.com/langgenius/dify-cloud-kit/oss/local"
"github.com/langgenius/dify-cloud-kit/oss/s3"
"github.com/langgenius/dify-cloud-kit/oss/tencentcos"
@@ -38,9 +38,9 @@ var OSSFactory = map[string]func(oss.OSSArgs) (oss.OSS, error){
"google-storage": gcsblob.NewGoogleCloudStorage,
"google_storage": gcsblob.NewGoogleCloudStorage,
"huawei": huanweiobs.NewHuaweiOBSStorage,
"huawei-obs": huanweiobs.NewHuaweiOBSStorage,
"huawei_obs": huanweiobs.NewHuaweiOBSStorage,
"huawei": huaweiobs.NewHuaweiOBSStorage,
"huawei-obs": huaweiobs.NewHuaweiOBSStorage,
"huawei_obs": huaweiobs.NewHuaweiOBSStorage,
"volcengine": volcenginetos.NewVolcengineTOSStorage,
"volcengine_tos": volcenginetos.NewVolcengineTOSStorage,
+17 -8
View File
@@ -29,13 +29,22 @@ func NewGoogleCloudStorage(args oss.OSSArgs) (oss.OSS, error) {
ctx := context.Background()
bucket := args.GoogleCloudStorage.Bucket
credentialsB64 := args.GoogleCloudStorage.CredentialsB64
credentials, err := base64.StdEncoding.DecodeString(credentialsB64)
if err != nil {
return nil, oss.ErrProviderInit.WithError(err).WithDetail("credentials must be a base64 encoded string")
}
client, err := storage.NewClient(ctx, option.WithCredentialsJSON(credentials))
if err != nil {
return nil, oss.ErrProviderInit.WithError(err)
var client *storage.Client
if credentialsB64 == "" {
client, err = storage.NewClient(ctx)
if err != nil {
return nil, oss.ErrProviderInit.WithError(err).WithDetail("failed to initialize GCS client with ADC")
}
} else {
credentials, err := base64.StdEncoding.DecodeString(credentialsB64)
if err != nil {
return nil, oss.ErrProviderInit.WithError(err).WithDetail("credentials must be a base64 encoded string")
}
client, err = storage.NewClient(ctx, option.WithCredentialsJSON(credentials))
if err != nil {
return nil, oss.ErrProviderInit.WithError(err)
}
}
return &GoogleCloudStorage{
bucket: bucket,
@@ -116,7 +125,7 @@ func (g *GoogleCloudStorage) List(prefix string) ([]oss.OSSPath, error) {
key = strings.TrimPrefix(key, "/")
res = append(res, oss.OSSPath{
Path: attrs.Name,
Path: key,
IsDir: false,
})
+34 -21
View File
@@ -1,4 +1,4 @@
package huanweiobs
package huaweiobs
import (
"io"
@@ -115,30 +115,43 @@ func (h *HuaweiOBSStorage) State(key string) (oss.OSSState, error) {
}
func (h *HuaweiOBSStorage) List(prefix string) ([]oss.OSSPath, error) {
output, err := h.client.ListObjects(&obs.ListObjectsInput{
Bucket: h.bucket,
ListObjsInput: obs.ListObjsInput{
Prefix: prefix,
},
})
if err != nil {
return nil, err
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
paths := []oss.OSSPath{}
for _, v := range output.Contents {
key := strings.TrimPrefix(v.Key, prefix)
key = strings.TrimPrefix(key, "/")
if key == "" {
continue
}
paths = append(paths, oss.OSSPath{
Path: key,
IsDir: func() bool {
return strings.HasSuffix(v.Key, "/")
}(),
marker := ""
paths := []oss.OSSPath{}
for {
output, err := h.client.ListObjects(&obs.ListObjectsInput{
Bucket: h.bucket,
ListObjsInput: obs.ListObjsInput{
Prefix: prefix,
},
Marker: marker,
})
if err != nil {
return nil, err
}
for _, v := range output.Contents {
key := strings.TrimPrefix(v.Key, prefix)
key = strings.TrimPrefix(key, "/")
if key == "" {
continue
}
paths = append(paths, oss.OSSPath{
Path: key,
IsDir: false,
})
}
if output.IsTruncated {
marker = output.NextMarker
} else {
break
}
}
return paths, nil
}
+21 -7
View File
@@ -72,22 +72,35 @@ type S3 struct {
func (s *S3) Validate() error {
if s.Bucket == "" || s.Region == "" {
msg := fmt.Sprintf("bucket and region cannot be empty.")
msg := "bucket and region cannot be empty."
return ErrArgumentInvalid.WithDetail(msg)
}
return nil
}
type AzureBlob struct {
ConnectionString string
ContainerName string
ConnectionString string
ContainerName string
UseManagedIdentity bool
ServiceURL string
}
func (a *AzureBlob) Validate() error {
if a.ConnectionString == "" || a.ContainerName == "" {
msg := fmt.Sprintf("connectorString and containerName cannot be empty.")
if a.ContainerName == "" {
msg := "containerName cannot be empty"
return ErrArgumentInvalid.WithDetail(msg)
}
if a.UseManagedIdentity {
if a.ServiceURL == "" {
msg := "service url cannot be empty. e.g. https://<account>.blob.core.windows.net/."
return ErrArgumentInvalid.WithDetail(msg)
}
} else {
if a.ConnectionString == "" {
msg := "connectorString cannot be empty."
return ErrArgumentInvalid.WithDetail(msg)
}
}
return nil
}
@@ -126,6 +139,7 @@ type TencentCOS struct {
SecretID string
SecretKey string
Bucket string
Endpoint string
}
func (t *TencentCOS) Validate() error {
@@ -142,8 +156,8 @@ type GoogleCloudStorage struct {
}
func (g *GoogleCloudStorage) Validate() error {
if g.Bucket == "" || g.CredentialsB64 == "" {
msg := fmt.Sprintf("bucket and credentials cannot be empty.")
if g.Bucket == "" {
msg := fmt.Sprintf("bucket cannot be empty.")
return ErrArgumentInvalid.WithDetail(msg)
}
return nil
+5 -1
View File
@@ -33,7 +33,11 @@ func NewTencentCOSStorage(args oss.OSSArgs) (oss.OSS, error) {
region := args.TencentCOS.Region
secretID := args.TencentCOS.SecretID
secretKey := args.TencentCOS.SecretKey
u, err := url.Parse("https://" + bucket + ".cos." + region + ".myqcloud.com")
endpoint := args.TencentCOS.Endpoint
if endpoint == "" {
endpoint = "https://" + bucket + ".cos." + region + ".myqcloud.com"
}
u, err := url.Parse(endpoint)
if err != nil {
return nil, oss.ErrProviderInit.WithError(err).WithDetail("url parse failed")
}
+14
View File
@@ -118,6 +118,17 @@ var allCases = []testArgsCases{
},
skip: false,
},
{
vendor: "azure",
args: oss.OSSArgs{
AzureBlob: &oss.AzureBlob{
UseManagedIdentity: true,
ContainerName: os.Getenv("AZURE_CONTAINER"),
ServiceURL: os.Getenv("AZURE_SERVICE_URL"),
},
},
skip: true,
},
}
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
@@ -174,6 +185,9 @@ func TestAll(t *testing.T) {
ossPaths, err = storage.List(prefix)
assert.Equal(t, 1, len(ossPaths), info)
assert.Nil(t, err, info)
// Verify that List returns relative paths (without prefix)
expectedRelativePath := key[len(prefix)+1:] // +1 for the "/" separator
assert.Equal(t, expectedRelativePath, ossPaths[0].Path, info+" - List should return relative path")
err = storage.Delete(key)
assert.Nil(t, err, info)