mirror of
https://github.com/langgenius/dify-cloud-kit.git
synced 2026-07-26 05:54:49 -04:00
Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3c6b12065a | |||
| 4adc4497ae | |||
| f932ae4155 | |||
| bdc2eef3cd | |||
| b5d897642a | |||
| d56e719c49 | |||
| c4a09cd10a | |||
| f560f324a2 | |||
| dd700497ef | |||
| 0a0e117e58 | |||
| ec849b2202 | |||
| b8203dccda | |||
| d4b8e7c17e | |||
| c56bbba411 | |||
| 2b8f3664d5 | |||
| c54203d9e9 | |||
| 1b8f6a174d | |||
| 681efb7762 | |||
| 553b38edd4 | |||
| 4ffc7bdc65 |
@@ -1,2 +1,111 @@
|
||||
# go-cloud-kit
|
||||
A library and tools for open cloud development in Go.
|
||||
# Dify Cloud Kit
|
||||
|
||||
Dify Cloud Kit is a unified abstraction library for integrating various cloud object storage services in Go. It simplifies switching between providers and supports local testing and multi-cloud deployments.
|
||||
|
||||
## ✨ Features
|
||||
|
||||
- Supports multiple backends: Local FS, Aliyun OSS, AWS S3, Azure Blob, Tencent COS, Huawei OBS, Google GCS
|
||||
- Unified and clean interface
|
||||
- Factory pattern to dynamically load drivers
|
||||
- Easy to write tests with local and in-memory backends
|
||||
|
||||
## 📦 Installation
|
||||
|
||||
```bash
|
||||
go get github.com/langgenius/dify-cloud-kit
|
||||
```
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
```go
|
||||
import (
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/factory"
|
||||
)
|
||||
|
||||
func main() {
|
||||
store, err := factory.Load("local", oss.OSSArgs{
|
||||
Local: &oss.Local{
|
||||
Path: "/tmp/files",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
files, _ := store.List("/")
|
||||
fmt.Println(files)
|
||||
}
|
||||
```
|
||||
|
||||
## 📁 Supported Storage Providers
|
||||
|
||||
| Provider | Module Path | Required Fields |
|
||||
|----------------|----------------------------|-------------------------------------------------|
|
||||
| Local | `oss/local/localfile.go` | `Path` |
|
||||
| Aliyun OSS | `oss/aliyun/aliyun.go` | `Endpoint`, `AccessKey`, `SecretKey`, `Bucket` |
|
||||
| AWS S3 | `oss/s3/s3.go` | `Region`, `AccessKey`, `SecretKey`, `Bucket` |
|
||||
| Azure Blob | `oss/azureblob/blob.go` | `AccountName`, `AccountKey`, `Container` |
|
||||
| Google GCS | `oss/gcsblob/gcs.go` | `CredentialsJSON`, `Bucket` |
|
||||
| Tencent COS | `oss/tencentcos/cos.go` | `SecretId`, `SecretKey`, `Bucket`, `Region` |
|
||||
| Huawei OBS | `oss/huaweiobs/obs.go` | `AK`, `SK`, `Endpoint`, `Bucket` |
|
||||
| Volcengine TOS | `oss/volcenginetos/tos.go` | `Endpoint`, `AccessKey`, `SecretKey`, `Bucket` |
|
||||
|
||||
## 🏗️ Usage with Factory
|
||||
|
||||
You can dynamically load a storage backend using the factory:
|
||||
|
||||
```go
|
||||
store, err := factory.Load("s3", oss.OSSArgs{
|
||||
S3: &oss.S3{
|
||||
Region: "us-west-2",
|
||||
AccessKey: "AKIA...",
|
||||
SecretKey: "SECRET...",
|
||||
Bucket: "my-bucket",
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
## 🧪 Testing
|
||||
|
||||
Unit tests are located in `tests/oss/oss_test.go`.
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Some providers require credentials to be passed via environment variables. Set them as needed:
|
||||
|
||||
```bash
|
||||
export OSS_AWS_ACCESS_KEY=your-access-key
|
||||
export OSS_AWS_SECRET_KEY=your-secret-key
|
||||
export OSS_AWS_REGION=your-region
|
||||
export OSS_S3_BUCKET=test-bucket
|
||||
|
||||
export OSS_ALIYUN_ENDPOINT=your-endpoint
|
||||
export OSS_ALIYUN_ACCESS_KEY=your-access-key
|
||||
export OSS_ALIYUN_SECRET_KEY=your-secret-key
|
||||
export OSS_ALIYUN_BUCKET=test-bucket
|
||||
|
||||
```
|
||||
|
||||
### Run Tests
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
```
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the [Apache 2.0 License](LICENSE).
|
||||
|
||||
## NOTICE
|
||||
Some parts of the code in this project originate from [dify-plugin-daemon](https://github.com/langgenius/dify-plugin-daemon)
|
||||
|
||||
| Provider | Author | PR |
|
||||
|----------------|---|---|
|
||||
| Aliyun OSS |[bravomark](https://github.com/bravomark)|https://github.com/langgenius/dify-plugin-daemon/pull/261 |
|
||||
| Azure Blob |[techan](https://github.com/te-chan)|https://github.com/langgenius/dify-plugin-daemon/pull/172|
|
||||
| Google GCS |[Hironori Yamamoto](https://github.com/hiro-o918)|https://github.com/langgenius/dify-plugin-daemon/pull/237|
|
||||
| Local |[lengyhua](https://github.com/lengyhua)|https://github.com/langgenius/dify-plugin-daemon/pull/157|
|
||||
| AWS S3 |[Yeuoly](https://github.com/Yeuoly)|https://github.com/langgenius/dify-plugin-daemon/commit/9ad9d7d4de1d123956ab07955e541bc4053e5170|
|
||||
| Tencent COS |[quicksand](https://github.com/quicksandznzn)|https://github.com/langgenius/dify-plugin-daemon/pull/97|
|
||||
| Volcengine TOS |[quicksand](https://github.com/quicksandznzn)|https://github.com/langgenius/dify-cloud-kit/pull/2|
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
module github.com/langgenius/dify-cloud-kit
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go/storage v1.54.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.14
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.67
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3
|
||||
github.com/aws/smithy-go v1.24.2
|
||||
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.7.65
|
||||
github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.12
|
||||
google.golang.org/api v0.232.0
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go v0.121.0 // indirect
|
||||
cloud.google.com/go/auth v0.16.1 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.2 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/clbanning/mxj v1.8.4 // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
|
||||
github.com/google/go-querystring v1.0.0 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.14.1 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.3 // indirect
|
||||
github.com/mozillazg/go-httpheader v0.2.1 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
golang.org/x/crypto v0.52.0 // indirect
|
||||
golang.org/x/net v0.55.0 // indirect
|
||||
golang.org/x/oauth2 v0.34.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.45.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
golang.org/x/time v0.11.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
|
||||
google.golang.org/grpc v1.79.3 // indirect
|
||||
google.golang.org/protobuf v1.36.10 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -0,0 +1,239 @@
|
||||
cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
|
||||
cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
|
||||
cloud.google.com/go v0.121.0 h1:pgfwva8nGw7vivjZiRfrmglGWiCJBP+0OmDpenG/Fwg=
|
||||
cloud.google.com/go v0.121.0/go.mod h1:rS7Kytwheu/y9buoDmu5EIpMMCI4Mb8ND4aeN4Vwj7Q=
|
||||
cloud.google.com/go/auth v0.16.1 h1:XrXauHMd30LhQYVRHLGvJiYeczweKQXZxsTbV9TiguU=
|
||||
cloud.google.com/go/auth v0.16.1/go.mod h1:1howDHJ5IETh/LwYs3ZxvlkXF48aSqqJUM+5o02dNOI=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
|
||||
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
|
||||
cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8=
|
||||
cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE=
|
||||
cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc=
|
||||
cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA=
|
||||
cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE=
|
||||
cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY=
|
||||
cloud.google.com/go/monitoring v1.24.0 h1:csSKiCJ+WVRgNkRzzz3BPoGjFhjPY23ZTcaenToJxMM=
|
||||
cloud.google.com/go/monitoring v1.24.0/go.mod h1:Bd1PRK5bmQBQNnuGwHBfUamAV1ys9049oEPHnn4pcsc=
|
||||
cloud.google.com/go/storage v1.54.0 h1:Du3XEyliAiftfyW0bwfdppm2MMLdpVAfiIg4T2nAI+0=
|
||||
cloud.google.com/go/storage v1.54.0/go.mod h1:hIi9Boe8cHxTyaeqh7KMMwKg088VblFK46C2x/BWaZE=
|
||||
cloud.google.com/go/trace v1.11.3 h1:c+I4YFjxRQjvAhRmSsmjpASUKq88chOX854ied0K/pE=
|
||||
cloud.google.com/go/trace v1.11.3/go.mod h1:pt7zCYiDSQjC9Y2oqCsh9jF4GStB/hmjrYLsxRR27q8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0=
|
||||
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
|
||||
github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
|
||||
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w=
|
||||
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA=
|
||||
github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98=
|
||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
|
||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
|
||||
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
|
||||
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
|
||||
github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q=
|
||||
github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA=
|
||||
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible h1:yNjwdvn9fwuN6Ouxr0xHM0cVu03YMUWUyFmu2van/Yc=
|
||||
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
|
||||
github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
|
||||
github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
|
||||
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
|
||||
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM=
|
||||
github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0=
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.7.65 h1:+WBbfwThfZSbxpf1Dw6fyMwyzVtWBBExqfDJ5giiR2s=
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.7.65/go.mod h1:8+hG+mQMuRP/OIS9d83syAvXvrMj9HhkND6Q1fLghw0=
|
||||
github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.12 h1:u9+32DXQIOFPG8oQ3xrjSAUSyAcaq5bqO4cEBom/6lA=
|
||||
github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.12/go.mod h1:IrjK84IJJTuOZOTMv/P18Ydjy/x+ow7fF7q11jAxXLM=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||
golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
|
||||
golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
|
||||
golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/api v0.232.0 h1:qGnmaIMf7KcuwHOlF3mERVzChloDYwRfOJOrHt8YC3I=
|
||||
google.golang.org/api v0.232.0/go.mod h1:p9QCfBWZk1IJETUdbTKloR5ToFdKbYh2fkjsUL6vNoY=
|
||||
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE=
|
||||
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
|
||||
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,208 @@
|
||||
package aliyun
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
difyoss "github.com/langgenius/dify-cloud-kit/oss"
|
||||
)
|
||||
|
||||
type AliyunOSSStorage struct {
|
||||
client *oss.Client
|
||||
bucket *oss.Bucket
|
||||
path string
|
||||
}
|
||||
|
||||
func NewAliyunOSSStorage(args difyoss.OSSArgs) (difyoss.OSS, error) {
|
||||
var err error
|
||||
if args.AliyunOSS == nil {
|
||||
return nil, difyoss.ErrArgumentInvalid.WithDetail("can't find Aiyun OSS argument in OSSArgs")
|
||||
}
|
||||
|
||||
err = args.AliyunOSS.Validate()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
region := args.AliyunOSS.Region
|
||||
endpoint := args.AliyunOSS.Endpoint
|
||||
accessKeyID := args.AliyunOSS.AccessKey
|
||||
accessKeySecret := args.AliyunOSS.SecretKey
|
||||
authVersion := args.AliyunOSS.AuthVersion
|
||||
path := args.AliyunOSS.Path
|
||||
bucketName := args.AliyunOSS.Bucket
|
||||
cloudboxId := args.AliyunOSS.CloudBoxId
|
||||
|
||||
// options
|
||||
var options []oss.ClientOption
|
||||
|
||||
// set region (required for v4)
|
||||
if region != "" {
|
||||
options = append(options, oss.Region(region))
|
||||
}
|
||||
|
||||
if cloudboxId != "" {
|
||||
options = append(options, oss.CloudBoxId(cloudboxId))
|
||||
}
|
||||
|
||||
// set auth-version
|
||||
if authVersion == "v1" {
|
||||
options = append(options, oss.AuthVersion(oss.AuthV1))
|
||||
} else if authVersion == "v4" {
|
||||
options = append(options, oss.AuthVersion(oss.AuthV4))
|
||||
} else {
|
||||
// default use v4
|
||||
options = append(options, oss.AuthVersion(oss.AuthV4))
|
||||
}
|
||||
|
||||
// create client
|
||||
var client *oss.Client
|
||||
|
||||
client, err = oss.New(endpoint, accessKeyID, accessKeySecret, options...)
|
||||
|
||||
if err != nil {
|
||||
return nil, difyoss.ErrProviderInit.WithError(err).WithDetail("failed to create Aliyun OSS client")
|
||||
}
|
||||
|
||||
// get specified bucket
|
||||
bucket, err := client.Bucket(bucketName)
|
||||
if err != nil {
|
||||
return nil, difyoss.ErrProviderInit.WithError(err).WithDetail(fmt.Sprintf("failed to get bucket %s", bucketName))
|
||||
}
|
||||
|
||||
// normalize path: remove leading slash, ensure trailing slash
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
if path != "" && !strings.HasSuffix(path, "/") {
|
||||
path = path + "/"
|
||||
}
|
||||
|
||||
return &AliyunOSSStorage{
|
||||
client: client,
|
||||
bucket: bucket,
|
||||
path: path,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// combine full object path
|
||||
func (s *AliyunOSSStorage) fullPath(key string) string {
|
||||
return path.Join(s.path, key)
|
||||
}
|
||||
|
||||
func (s *AliyunOSSStorage) Save(key string, data []byte) error {
|
||||
fullPath := s.fullPath(key)
|
||||
return s.bucket.PutObject(fullPath, bytes.NewReader(data))
|
||||
}
|
||||
|
||||
func (s *AliyunOSSStorage) Load(key string) ([]byte, error) {
|
||||
fullPath := s.fullPath(key)
|
||||
object, err := s.bucket.GetObject(fullPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Ensure object is closed after reading
|
||||
defer object.Close()
|
||||
|
||||
data, err := io.ReadAll(object)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (s *AliyunOSSStorage) Exists(key string) (bool, error) {
|
||||
fullPath := s.fullPath(key)
|
||||
return s.bucket.IsObjectExist(fullPath)
|
||||
}
|
||||
|
||||
func (s *AliyunOSSStorage) State(key string) (difyoss.OSSState, error) {
|
||||
fullPath := s.fullPath(key)
|
||||
meta, err := s.bucket.GetObjectMeta(fullPath)
|
||||
if err != nil {
|
||||
return difyoss.OSSState{}, err
|
||||
}
|
||||
|
||||
// Get content length
|
||||
size := int64(0)
|
||||
contentLength := meta.Get("Content-Length")
|
||||
if contentLength != "" {
|
||||
_, err := fmt.Sscanf(contentLength, "%d", &size)
|
||||
if err != nil {
|
||||
// Return zero size if parsing fails
|
||||
size = 0
|
||||
}
|
||||
}
|
||||
|
||||
// Get last modified time
|
||||
lastModified := time.Time{}
|
||||
lastModifiedStr := meta.Get("Last-Modified")
|
||||
if lastModifiedStr != "" {
|
||||
lastModified, err = time.Parse(time.RFC1123, lastModifiedStr)
|
||||
if err != nil {
|
||||
// Return zero time if parsing fails
|
||||
lastModified = time.Time{}
|
||||
}
|
||||
}
|
||||
|
||||
return difyoss.OSSState{
|
||||
Size: size,
|
||||
LastModified: lastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *AliyunOSSStorage) List(prefix string) ([]difyoss.OSSPath, error) {
|
||||
// combine given prefix with path
|
||||
fullPrefix := s.fullPath(prefix)
|
||||
|
||||
// Ensure the prefix ends with a slash for directories
|
||||
if !strings.HasSuffix(fullPrefix, "/") {
|
||||
fullPrefix = fullPrefix + "/"
|
||||
}
|
||||
|
||||
var keys []difyoss.OSSPath
|
||||
marker := ""
|
||||
for {
|
||||
lsRes, err := s.bucket.ListObjects(oss.Marker(marker), oss.Prefix(fullPrefix))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list objects in Aliyun OSS: %w", err)
|
||||
}
|
||||
|
||||
for _, object := range lsRes.Objects {
|
||||
if object.Key == fullPrefix {
|
||||
continue
|
||||
}
|
||||
// remove path and prefix from full path, only keep relative path
|
||||
key := strings.TrimPrefix(object.Key, fullPrefix)
|
||||
// Skip empty keys and directories (keys ending with /)
|
||||
if key == "" || strings.HasSuffix(key, "/") {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, difyoss.OSSPath{
|
||||
Path: key,
|
||||
IsDir: false,
|
||||
})
|
||||
}
|
||||
|
||||
// Check if there are more results
|
||||
if lsRes.IsTruncated {
|
||||
marker = lsRes.NextMarker
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (s *AliyunOSSStorage) Delete(key string) error {
|
||||
fullPath := s.fullPath(key)
|
||||
return s.bucket.DeleteObject(fullPath)
|
||||
}
|
||||
|
||||
func (s *AliyunOSSStorage) Type() string {
|
||||
return difyoss.OSS_TYPE_ALIYUN_OSS
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
)
|
||||
|
||||
type AzureBlobStorage struct {
|
||||
client *azblob.Client
|
||||
containerName string
|
||||
}
|
||||
|
||||
func NewAzureBlobStorage(args oss.OSSArgs) (oss.OSS, error) {
|
||||
var client *azblob.Client
|
||||
var err error
|
||||
|
||||
if args.AzureBlob == nil {
|
||||
return nil, oss.ErrArgumentInvalid.WithDetail("can't find Azure Blob argument in OSSArgs")
|
||||
}
|
||||
err = args.AzureBlob.Validate()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
containerName := args.AzureBlob.ContainerName
|
||||
if args.AzureBlob.UseManagedIdentity {
|
||||
cred, err := azidentity.NewDefaultAzureCredential(nil)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
client, err = azblob.NewClient(args.AzureBlob.ServiceURL, cred, nil)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
} else {
|
||||
connectionString := args.AzureBlob.ConnectionString
|
||||
client, err = azblob.NewClientFromConnectionString(connectionString, nil)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
|
||||
}
|
||||
return &AzureBlobStorage{
|
||||
client: client,
|
||||
containerName: containerName,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Save(key string, data []byte) error {
|
||||
_, err := a.client.UploadBuffer(context.TODO(), a.containerName, key, data, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Load(key string) ([]byte, error) {
|
||||
get, err := a.client.DownloadStream(context.TODO(), a.containerName, key, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
downloadedData := bytes.Buffer{}
|
||||
retryReader := get.NewRetryReader(context.TODO(), &azblob.RetryReaderOptions{})
|
||||
_, err = downloadedData.ReadFrom(retryReader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = retryReader.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return downloadedData.Bytes(), nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Exists(key string) (bool, error) {
|
||||
blobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlobClient(key)
|
||||
_, err := blobClient.GetProperties(context.TODO(), nil)
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "404") {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) State(key string) (oss.OSSState, error) {
|
||||
blobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlobClient(key)
|
||||
props, err := blobClient.GetProperties(context.TODO(), nil)
|
||||
|
||||
if err != nil {
|
||||
return oss.OSSState{}, err
|
||||
}
|
||||
|
||||
return oss.OSSState{
|
||||
Size: *props.ContentLength,
|
||||
LastModified: *props.LastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) List(prefix string) ([]oss.OSSPath, error) {
|
||||
// append a slash to the prefix if it doesn't end with one
|
||||
if !strings.HasSuffix(prefix, "/") {
|
||||
prefix = prefix + "/"
|
||||
}
|
||||
|
||||
pager := a.client.NewListBlobsFlatPager(a.containerName, &azblob.ListBlobsFlatOptions{
|
||||
Prefix: &prefix,
|
||||
})
|
||||
|
||||
paths := make([]oss.OSSPath, 0)
|
||||
for pager.More() {
|
||||
page, err := pager.NextPage(context.TODO())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, blob := range page.Segment.BlobItems {
|
||||
// remove prefix
|
||||
key := strings.TrimPrefix(*blob.Name, prefix)
|
||||
// remove leading slash
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
paths = append(paths, oss.OSSPath{
|
||||
Path: key,
|
||||
IsDir: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Delete(key string) error {
|
||||
_, err := a.client.DeleteBlob(context.TODO(), a.containerName, key, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *AzureBlobStorage) Type() string {
|
||||
return oss.OSS_TYPE_AZURE_BLOB
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package oss
|
||||
|
||||
import "fmt"
|
||||
|
||||
var (
|
||||
ErrProviderNotFound = NewCloudKitError("provider not found", "")
|
||||
ErrArgumentInvalid = NewCloudKitError("argument invalid", "")
|
||||
ErrProviderInit = NewCloudKitError("provider init error", "")
|
||||
)
|
||||
|
||||
type CloudKitError struct {
|
||||
Reason string
|
||||
Detail string
|
||||
Err error
|
||||
}
|
||||
|
||||
func NewCloudKitError(reason string, detail string) *CloudKitError {
|
||||
return &CloudKitError{
|
||||
Reason: reason,
|
||||
Detail: detail,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CloudKitError) Error() string {
|
||||
if c.Detail != "" {
|
||||
return fmt.Sprintf("reason: %s; detail: %s; error: %v", c.Reason, c.Detail, c.Err)
|
||||
}
|
||||
return fmt.Sprintf("reason: %s; error: %v", c.Reason, c.Err)
|
||||
}
|
||||
|
||||
func (c *CloudKitError) WithDetail(detail string) *CloudKitError {
|
||||
c.Detail = detail
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *CloudKitError) WithError(err error) *CloudKitError {
|
||||
c.Err = err
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package factory
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/aliyun"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/azureblob"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/gcsblob"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/huaweiobs"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/local"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/s3"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/tencentcos"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/volcenginetos"
|
||||
)
|
||||
|
||||
var OSSFactory = map[string]func(oss.OSSArgs) (oss.OSS, error){
|
||||
"local": local.NewLocalStorage,
|
||||
"local_file": local.NewLocalStorage,
|
||||
|
||||
"s3": s3.NewS3Storage,
|
||||
"aws_s3": s3.NewS3Storage,
|
||||
"aws-s3": s3.NewS3Storage,
|
||||
|
||||
"azure": azureblob.NewAzureBlobStorage,
|
||||
"azure_blob": azureblob.NewAzureBlobStorage,
|
||||
"azure-blob": azureblob.NewAzureBlobStorage,
|
||||
|
||||
"aliyun": aliyun.NewAliyunOSSStorage,
|
||||
"aliyun-oss": aliyun.NewAliyunOSSStorage,
|
||||
"aliyun_oss": aliyun.NewAliyunOSSStorage,
|
||||
|
||||
"tencent": tencentcos.NewTencentCOSStorage,
|
||||
"tencent_cos": tencentcos.NewTencentCOSStorage,
|
||||
"tencent-cos": tencentcos.NewTencentCOSStorage,
|
||||
|
||||
"gcs": gcsblob.NewGoogleCloudStorage,
|
||||
"google-storage": gcsblob.NewGoogleCloudStorage,
|
||||
"google_storage": gcsblob.NewGoogleCloudStorage,
|
||||
|
||||
"huawei": huaweiobs.NewHuaweiOBSStorage,
|
||||
"huawei-obs": huaweiobs.NewHuaweiOBSStorage,
|
||||
"huawei_obs": huaweiobs.NewHuaweiOBSStorage,
|
||||
|
||||
"volcengine": volcenginetos.NewVolcengineTOSStorage,
|
||||
"volcengine_tos": volcenginetos.NewVolcengineTOSStorage,
|
||||
"volcengine-tos": volcenginetos.NewVolcengineTOSStorage,
|
||||
}
|
||||
|
||||
func Load(name string, args oss.OSSArgs) (oss.OSS, error) {
|
||||
f, ok := OSSFactory[name]
|
||||
if !ok {
|
||||
msg := fmt.Sprintf("[ %s ] is not in the provider list", name)
|
||||
return nil, oss.ErrProviderNotFound.WithDetail(msg)
|
||||
}
|
||||
return f(args)
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package gcsblob
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
"google.golang.org/api/iterator"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
type GoogleCloudStorage struct {
|
||||
bucket string
|
||||
client *storage.Client
|
||||
}
|
||||
|
||||
func NewGoogleCloudStorage(args oss.OSSArgs) (oss.OSS, error) {
|
||||
if args.GoogleCloudStorage == nil {
|
||||
return nil, oss.ErrArgumentInvalid.WithDetail("can't find Google Cloud Storage argument in OSSArgs")
|
||||
}
|
||||
err := args.GoogleCloudStorage.Validate()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx := context.Background()
|
||||
bucket := args.GoogleCloudStorage.Bucket
|
||||
credentialsB64 := args.GoogleCloudStorage.CredentialsB64
|
||||
|
||||
var client *storage.Client
|
||||
if credentialsB64 == "" {
|
||||
client, err = storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err).WithDetail("failed to initialize GCS client with ADC")
|
||||
}
|
||||
} else {
|
||||
credentials, err := base64.StdEncoding.DecodeString(credentialsB64)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err).WithDetail("credentials must be a base64 encoded string")
|
||||
}
|
||||
client, err = storage.NewClient(ctx, option.WithCredentialsJSON(credentials))
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
}
|
||||
return &GoogleCloudStorage{
|
||||
bucket: bucket,
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (g *GoogleCloudStorage) Save(key string, data []byte) error {
|
||||
ctx := context.Background()
|
||||
obj := g.client.Bucket(g.bucket).Object(key)
|
||||
|
||||
wc := obj.NewWriter(ctx)
|
||||
if _, err := wc.Write(data); err != nil {
|
||||
return err
|
||||
}
|
||||
return wc.Close()
|
||||
}
|
||||
|
||||
func (g *GoogleCloudStorage) Load(key string) ([]byte, error) {
|
||||
rc, err := g.client.Bucket(g.bucket).Object(key).NewReader(context.Background())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
data, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (g *GoogleCloudStorage) Exists(key string) (bool, error) {
|
||||
obj := g.client.Bucket(g.bucket).Object(key)
|
||||
|
||||
_, err := obj.Attrs(context.Background())
|
||||
if err != nil {
|
||||
if errors.Is(err, storage.ErrObjectNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (g *GoogleCloudStorage) State(key string) (oss.OSSState, error) {
|
||||
obj := g.client.Bucket(g.bucket).Object(key)
|
||||
|
||||
attrs, err := obj.Attrs(context.Background())
|
||||
if err != nil {
|
||||
return oss.OSSState{}, err
|
||||
}
|
||||
return oss.OSSState{
|
||||
Size: attrs.Size,
|
||||
LastModified: attrs.Updated,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (g *GoogleCloudStorage) List(prefix string) ([]oss.OSSPath, error) {
|
||||
ctx := context.Background()
|
||||
it := g.client.Bucket(g.bucket).Objects(ctx, &storage.Query{
|
||||
Prefix: prefix,
|
||||
})
|
||||
res := []oss.OSSPath{}
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if attrs.Name == prefix {
|
||||
continue
|
||||
}
|
||||
|
||||
key := strings.TrimPrefix(attrs.Name, prefix)
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
|
||||
res = append(res, oss.OSSPath{
|
||||
Path: key,
|
||||
IsDir: false,
|
||||
})
|
||||
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (g *GoogleCloudStorage) Delete(key string) error {
|
||||
ctx := context.Background()
|
||||
obj := g.client.Bucket(g.bucket).Object(key)
|
||||
attrs, err := obj.Attrs(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
obj = obj.If(storage.Conditions{GenerationMatch: attrs.Generation})
|
||||
|
||||
return obj.Delete(ctx)
|
||||
}
|
||||
|
||||
func (g *GoogleCloudStorage) Type() string {
|
||||
return oss.OSS_TYPE_GCS
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package huaweiobs
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
)
|
||||
|
||||
type HuaweiOBSStorage struct {
|
||||
bucket string
|
||||
client *obs.ObsClient
|
||||
}
|
||||
|
||||
func NewHuaweiOBSStorage(args oss.OSSArgs) (oss.OSS, error) {
|
||||
if args.HuaweiOBS == nil {
|
||||
return nil, oss.ErrArgumentInvalid.WithDetail("can't find Huawei OBS argument in OSSArgs")
|
||||
}
|
||||
|
||||
err := args.HuaweiOBS.Validate()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ak := args.HuaweiOBS.AccessKey
|
||||
sk := args.HuaweiOBS.SecretKey
|
||||
endpoint := args.HuaweiOBS.Server
|
||||
bucket := args.HuaweiOBS.Bucket
|
||||
pathStyle := args.HuaweiOBS.PathStyle
|
||||
client, err := obs.New(ak, sk, endpoint, obs.WithPathStyle(pathStyle))
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
|
||||
return &HuaweiOBSStorage{
|
||||
bucket: bucket,
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (h *HuaweiOBSStorage) Save(key string, data []byte) error {
|
||||
tmpFilename := randomString(5)
|
||||
file, err := os.CreateTemp("/tmp", tmpFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
|
||||
_, err = file.Write(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = h.client.PutFile(&obs.PutFileInput{
|
||||
PutObjectBasicInput: obs.PutObjectBasicInput{
|
||||
ObjectOperationInput: obs.ObjectOperationInput{
|
||||
Bucket: h.bucket,
|
||||
Key: key,
|
||||
},
|
||||
},
|
||||
SourceFile: file.Name(),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (h *HuaweiOBSStorage) Load(key string) ([]byte, error) {
|
||||
output, err := h.client.GetObject(&obs.GetObjectInput{
|
||||
GetObjectMetadataInput: obs.GetObjectMetadataInput{
|
||||
Bucket: h.bucket,
|
||||
Key: key,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer output.Body.Close()
|
||||
|
||||
return io.ReadAll(output.Body)
|
||||
}
|
||||
|
||||
func (h *HuaweiOBSStorage) Exists(key string) (bool, error) {
|
||||
_, err := h.client.HeadObject(&obs.HeadObjectInput{
|
||||
Bucket: h.bucket,
|
||||
Key: key,
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if obsErr, ok := err.(obs.ObsError); ok && obsErr.StatusCode == 404 {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
func (h *HuaweiOBSStorage) State(key string) (oss.OSSState, error) {
|
||||
output, err := h.client.GetAttribute(&obs.GetAttributeInput{
|
||||
GetObjectMetadataInput: obs.GetObjectMetadataInput{
|
||||
Bucket: h.bucket,
|
||||
Key: key,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return oss.OSSState{}, err
|
||||
}
|
||||
return oss.OSSState{
|
||||
Size: output.ContentLength,
|
||||
LastModified: output.LastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (h *HuaweiOBSStorage) List(prefix string) ([]oss.OSSPath, error) {
|
||||
if !strings.HasSuffix(prefix, "/") {
|
||||
prefix = prefix + "/"
|
||||
}
|
||||
|
||||
marker := ""
|
||||
paths := []oss.OSSPath{}
|
||||
for {
|
||||
output, err := h.client.ListObjects(&obs.ListObjectsInput{
|
||||
Bucket: h.bucket,
|
||||
ListObjsInput: obs.ListObjsInput{
|
||||
Prefix: prefix,
|
||||
},
|
||||
Marker: marker,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, v := range output.Contents {
|
||||
key, ok := listedObjectPath(prefix, v.Key)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
paths = append(paths, oss.OSSPath{
|
||||
Path: key,
|
||||
IsDir: false,
|
||||
})
|
||||
}
|
||||
if output.IsTruncated {
|
||||
marker = output.NextMarker
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
func (h *HuaweiOBSStorage) Delete(key string) error {
|
||||
_, err := h.client.DeleteObject(&obs.DeleteObjectInput{
|
||||
Bucket: h.bucket,
|
||||
Key: key,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (h *HuaweiOBSStorage) Type() string {
|
||||
return oss.OSS_TYPE_HUAWEI_OBS
|
||||
}
|
||||
|
||||
func listedObjectPath(prefix string, objectKey string) (string, bool) {
|
||||
key := strings.TrimPrefix(objectKey, prefix)
|
||||
key = strings.TrimLeft(key, "/")
|
||||
if key == "" || strings.HasSuffix(key, "/") {
|
||||
return "", false
|
||||
}
|
||||
|
||||
return key, true
|
||||
}
|
||||
|
||||
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
|
||||
func randomString(length int) string {
|
||||
seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
result := make([]byte, length)
|
||||
for i := range result {
|
||||
result[i] = charset[seededRand.Intn(len(charset))]
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package huaweiobs
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestListedObjectPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prefix string
|
||||
objectKey string
|
||||
want string
|
||||
wantOK bool
|
||||
}{
|
||||
{
|
||||
name: "file under prefix",
|
||||
prefix: "plugins/",
|
||||
objectKey: "plugins/langgenius/model-provider.difypkg",
|
||||
want: "langgenius/model-provider.difypkg",
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "file under prefix without trailing slash",
|
||||
prefix: "plugins",
|
||||
objectKey: "plugins/langgenius/model-provider.difypkg",
|
||||
want: "langgenius/model-provider.difypkg",
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "directory placeholder under prefix",
|
||||
prefix: "plugins/",
|
||||
objectKey: "plugins/langgenius/",
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "prefix placeholder",
|
||||
prefix: "plugins/",
|
||||
objectKey: "plugins/",
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "leading slash after prefix",
|
||||
prefix: "plugins",
|
||||
objectKey: "plugins//langgenius/model-provider.difypkg",
|
||||
want: "langgenius/model-provider.difypkg",
|
||||
wantOK: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := listedObjectPath(tt.prefix, tt.objectKey)
|
||||
if ok != tt.wantOK {
|
||||
t.Fatalf("listedObjectPath() ok = %v, want %v", ok, tt.wantOK)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("listedObjectPath() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
)
|
||||
|
||||
type LocalStorage struct {
|
||||
root string
|
||||
}
|
||||
|
||||
func NewLocalStorage(args oss.OSSArgs) (oss.OSS, error) {
|
||||
if args.Local == nil {
|
||||
return nil, oss.ErrArgumentInvalid.WithDetail("can't find Local argument in OSSArgs")
|
||||
}
|
||||
err := args.Local.Validate()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root := args.Local.Path
|
||||
if err := os.MkdirAll(root, 0755); err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err).WithDetail("failed to create storage path")
|
||||
}
|
||||
|
||||
return &LocalStorage{root: root}, nil
|
||||
}
|
||||
|
||||
func (l *LocalStorage) Save(key string, data []byte) error {
|
||||
path := filepath.Join(l.root, key)
|
||||
filePath := filepath.Dir(path)
|
||||
if err := os.MkdirAll(filePath, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.WriteFile(path, data, 0o644)
|
||||
}
|
||||
|
||||
func (l *LocalStorage) Load(key string) ([]byte, error) {
|
||||
path := filepath.Join(l.root, key)
|
||||
|
||||
return os.ReadFile(path)
|
||||
}
|
||||
|
||||
func (l *LocalStorage) Exists(key string) (bool, error) {
|
||||
path := filepath.Join(l.root, key)
|
||||
|
||||
_, err := os.Stat(path)
|
||||
return err == nil, nil
|
||||
}
|
||||
|
||||
func (l *LocalStorage) State(key string) (oss.OSSState, error) {
|
||||
path := filepath.Join(l.root, key)
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return oss.OSSState{}, err
|
||||
}
|
||||
|
||||
return oss.OSSState{Size: info.Size(), LastModified: info.ModTime()}, nil
|
||||
}
|
||||
|
||||
func (l *LocalStorage) List(prefix string) ([]oss.OSSPath, error) {
|
||||
paths := make([]oss.OSSPath, 0)
|
||||
// check if the patch exists
|
||||
exists, err := l.Exists(prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !exists {
|
||||
return paths, nil
|
||||
}
|
||||
prefix = filepath.Join(l.root, prefix)
|
||||
|
||||
err = filepath.WalkDir(prefix, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// remove prefix
|
||||
path = strings.TrimPrefix(path, prefix)
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
// remove leading slash
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
paths = append(paths, oss.OSSPath{
|
||||
Path: path,
|
||||
IsDir: d.IsDir(),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
func (l *LocalStorage) Delete(key string) error {
|
||||
path := filepath.Join(l.root, key)
|
||||
|
||||
return os.RemoveAll(path)
|
||||
}
|
||||
|
||||
func (l *LocalStorage) Type() string {
|
||||
return oss.OSS_TYPE_LOCAL
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
package local
|
||||
+196
@@ -0,0 +1,196 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// OSS supports different types of object storage services
|
||||
// such as local file system, AWS S3, and Tencent COS.
|
||||
// The interface defines methods for saving, loading, checking existence,
|
||||
const (
|
||||
OSS_TYPE_LOCAL = "local"
|
||||
OSS_TYPE_S3 = "aws_s3"
|
||||
OSS_TYPE_TENCENT_COS = "tencent_cos"
|
||||
OSS_TYPE_AZURE_BLOB = "azure_blob"
|
||||
OSS_TYPE_GCS = "gcs"
|
||||
OSS_TYPE_ALIYUN_OSS = "aliyun_oss"
|
||||
OSS_TYPE_HUAWEI_OBS = "huawei_obs"
|
||||
OSS_TYPE_VOLCENGINE_TOS = "volcengine_tos"
|
||||
)
|
||||
|
||||
type OSSState struct {
|
||||
Size int64
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
type OSSPath struct {
|
||||
Path string
|
||||
IsDir bool
|
||||
}
|
||||
|
||||
type OSS interface {
|
||||
// Save saves data into path key
|
||||
Save(key string, data []byte) error
|
||||
// Load loads data from path key
|
||||
Load(key string) ([]byte, error)
|
||||
// Exists checks if the data exists in the path key
|
||||
Exists(key string) (bool, error)
|
||||
// State gets the state of the data in the path key
|
||||
State(key string) (OSSState, error)
|
||||
// List lists all the data with the given prefix, and all the paths are absolute paths
|
||||
List(prefix string) ([]OSSPath, error)
|
||||
// Delete deletes the data in the path key
|
||||
Delete(key string) error
|
||||
// Type returns the type of the storage
|
||||
// For example: local, aws_s3, tencent_cos
|
||||
Type() string
|
||||
}
|
||||
|
||||
type OSSArgs struct {
|
||||
S3 *S3
|
||||
Local *Local
|
||||
AzureBlob *AzureBlob
|
||||
AliyunOSS *AliyunOSS
|
||||
TencentCOS *TencentCOS
|
||||
GoogleCloudStorage *GoogleCloudStorage
|
||||
HuaweiOBS *HuaweiOBS
|
||||
VolcengineTOS *VolcengineTOS
|
||||
}
|
||||
|
||||
type S3 struct {
|
||||
UseAws bool
|
||||
Endpoint string
|
||||
UsePathStyle bool
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
Bucket string
|
||||
Region string
|
||||
UseIamRole bool
|
||||
SignatureVersion string
|
||||
}
|
||||
|
||||
func (s *S3) Validate() error {
|
||||
if s.Bucket == "" || s.Region == "" {
|
||||
msg := "bucket and region cannot be empty."
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type AzureBlob struct {
|
||||
ConnectionString string
|
||||
ContainerName string
|
||||
UseManagedIdentity bool
|
||||
ServiceURL string
|
||||
}
|
||||
|
||||
func (a *AzureBlob) Validate() error {
|
||||
if a.ContainerName == "" {
|
||||
msg := "containerName cannot be empty"
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
if a.UseManagedIdentity {
|
||||
if a.ServiceURL == "" {
|
||||
msg := "service url cannot be empty. e.g. https://<account>.blob.core.windows.net/."
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
} else {
|
||||
if a.ConnectionString == "" {
|
||||
msg := "connectorString cannot be empty."
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Local struct {
|
||||
Path string
|
||||
}
|
||||
|
||||
func (l *Local) Validate() error {
|
||||
if l.Path == "" {
|
||||
return ErrArgumentInvalid.WithDetail("path cannot be empty")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type AliyunOSS struct {
|
||||
Region string
|
||||
Endpoint string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
AuthVersion string
|
||||
Path string
|
||||
Bucket string
|
||||
CloudBoxId string
|
||||
}
|
||||
|
||||
func (a *AliyunOSS) Validate() error {
|
||||
if a.Bucket == "" || a.SecretKey == "" || a.AccessKey == "" || a.Endpoint == "" {
|
||||
msg := fmt.Sprintf("bucket, accesskKey, secretKey, endpoint cannot be empty.")
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type TencentCOS struct {
|
||||
Region string
|
||||
SecretID string
|
||||
SecretKey string
|
||||
Bucket string
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
func (t *TencentCOS) Validate() error {
|
||||
if t.Bucket == "" || t.Region == "" || t.SecretID == "" || t.SecretKey == "" {
|
||||
msg := fmt.Sprintf("bucket, region, secretKey, secretID cannot be empty.")
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type GoogleCloudStorage struct {
|
||||
Bucket string
|
||||
CredentialsB64 string
|
||||
}
|
||||
|
||||
func (g *GoogleCloudStorage) Validate() error {
|
||||
if g.Bucket == "" {
|
||||
msg := fmt.Sprintf("bucket cannot be empty.")
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type HuaweiOBS struct {
|
||||
Bucket string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
Server string
|
||||
PathStyle bool
|
||||
}
|
||||
|
||||
func (h *HuaweiOBS) Validate() error {
|
||||
if h.Bucket == "" || h.AccessKey == "" || h.SecretKey == "" || h.Server == "" {
|
||||
msg := fmt.Sprintf("bucket, accesskKey, secretKey, server cannot be empty.")
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type VolcengineTOS struct {
|
||||
Region string
|
||||
Endpoint string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
Bucket string
|
||||
}
|
||||
|
||||
func (t *VolcengineTOS) Validate() error {
|
||||
if t.Bucket == "" || t.Endpoint == "" || t.AccessKey == "" || t.SecretKey == "" {
|
||||
msg := fmt.Sprintf("bucket, endpoint,accessKey, secretKey cannot be empty.")
|
||||
return ErrArgumentInvalid.WithDetail(msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+235
@@ -0,0 +1,235 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
)
|
||||
|
||||
type S3Storage struct {
|
||||
bucket string
|
||||
client *s3.Client
|
||||
}
|
||||
|
||||
func NewS3Storage(args oss.OSSArgs) (oss.OSS, error) {
|
||||
var err error
|
||||
if args.S3 == nil {
|
||||
return nil, oss.ErrArgumentInvalid.WithDetail("can't find s3 argument in OSSArgs")
|
||||
}
|
||||
err = args.S3.Validate()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
useAws := args.S3.UseAws
|
||||
ak := args.S3.AccessKey
|
||||
sk := args.S3.SecretKey
|
||||
region := args.S3.Region
|
||||
endpoint := args.S3.Endpoint
|
||||
usePathStyle := args.S3.UsePathStyle
|
||||
bucket := args.S3.Bucket
|
||||
useIamRole := args.S3.UseIamRole
|
||||
signatureVersion := args.S3.SignatureVersion
|
||||
|
||||
var cfg aws.Config
|
||||
var client *s3.Client
|
||||
|
||||
if useAws {
|
||||
if (ak == "" && sk == "") || useIamRole {
|
||||
cfg, err = config.LoadDefaultConfig(
|
||||
context.TODO(),
|
||||
config.WithRegion(region),
|
||||
)
|
||||
} else {
|
||||
// 处理签名版本和凭证
|
||||
var credProvider aws.CredentialsProvider
|
||||
if strings.ToLower(signatureVersion) == "unsigned" {
|
||||
credProvider = aws.AnonymousCredentials{}
|
||||
} else {
|
||||
credProvider = credentials.NewStaticCredentialsProvider(
|
||||
ak,
|
||||
sk,
|
||||
"",
|
||||
)
|
||||
}
|
||||
|
||||
cfg, err = config.LoadDefaultConfig(
|
||||
context.TODO(),
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(credProvider),
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
|
||||
client = s3.NewFromConfig(cfg, func(options *s3.Options) {
|
||||
if endpoint != "" {
|
||||
options.BaseEndpoint = aws.String(endpoint)
|
||||
}
|
||||
options.UsePathStyle = usePathStyle
|
||||
})
|
||||
} else {
|
||||
var credProvider aws.CredentialsProvider
|
||||
if strings.ToLower(signatureVersion) == "unsigned" {
|
||||
credProvider = aws.AnonymousCredentials{}
|
||||
} else {
|
||||
credProvider = credentials.NewStaticCredentialsProvider(ak, sk, "")
|
||||
}
|
||||
client = s3.New(s3.Options{
|
||||
Credentials: credProvider,
|
||||
UsePathStyle: usePathStyle,
|
||||
Region: region,
|
||||
EndpointResolver: s3.EndpointResolverFunc(
|
||||
func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) {
|
||||
signingMethod := normalizeSignatureVersion(signatureVersion)
|
||||
return aws.Endpoint{
|
||||
URL: endpoint,
|
||||
HostnameImmutable: false,
|
||||
SigningName: "s3",
|
||||
PartitionID: "aws",
|
||||
SigningRegion: region,
|
||||
SigningMethod: signingMethod,
|
||||
Source: aws.EndpointSourceCustom,
|
||||
}, nil
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
// check bucket
|
||||
_, err = client.HeadBucket(context.TODO(), &s3.HeadBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
if err != nil {
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NotFound" {
|
||||
_, err = client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return &S3Storage{bucket: bucket, client: client}, nil
|
||||
}
|
||||
|
||||
func normalizeSignatureVersion(version string) string {
|
||||
switch strings.ToLower(version) {
|
||||
case "unsigned":
|
||||
return ""
|
||||
case "":
|
||||
return "v4" // 默认使用 v4
|
||||
default:
|
||||
return version
|
||||
}
|
||||
}
|
||||
|
||||
func (s *S3Storage) Save(key string, data []byte) error {
|
||||
_, err := s.client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader(data),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3Storage) Load(key string) ([]byte, error) {
|
||||
resp, err := s.client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
|
||||
func (s *S3Storage) Exists(key string) (bool, error) {
|
||||
_, err := s.client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
return err == nil, nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) Delete(key string) error {
|
||||
_, err := s.client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3Storage) List(prefix string) ([]oss.OSSPath, error) {
|
||||
// append a slash to the prefix if it doesn't end with one
|
||||
if !strings.HasSuffix(prefix, "/") {
|
||||
prefix = prefix + "/"
|
||||
}
|
||||
|
||||
var keys []oss.OSSPath
|
||||
input := &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Prefix: aws.String(prefix),
|
||||
}
|
||||
|
||||
paginator := s3.NewListObjectsV2Paginator(s.client, input)
|
||||
for paginator.HasMorePages() {
|
||||
page, err := paginator.NextPage(context.TODO())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, obj := range page.Contents {
|
||||
// remove prefix
|
||||
key := strings.TrimPrefix(*obj.Key, prefix)
|
||||
// remove leading slash
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
keys = append(keys, oss.OSSPath{
|
||||
Path: key,
|
||||
IsDir: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) State(key string) (oss.OSSState, error) {
|
||||
resp, err := s.client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return oss.OSSState{}, err
|
||||
}
|
||||
|
||||
if resp.ContentLength == nil {
|
||||
resp.ContentLength = ToPtr[int64](0)
|
||||
}
|
||||
if resp.LastModified == nil {
|
||||
resp.LastModified = ToPtr(time.Time{})
|
||||
}
|
||||
|
||||
return oss.OSSState{
|
||||
Size: *resp.ContentLength,
|
||||
LastModified: *resp.LastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) Type() string {
|
||||
return oss.OSS_TYPE_S3
|
||||
}
|
||||
|
||||
func ToPtr[T any](value T) *T {
|
||||
return &value
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package tencentcos
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
"github.com/tencentyun/cos-go-sdk-v5"
|
||||
)
|
||||
|
||||
type TencentCOSStorage struct {
|
||||
bucket string
|
||||
region string
|
||||
client *cos.Client
|
||||
}
|
||||
|
||||
func NewTencentCOSStorage(args oss.OSSArgs) (oss.OSS, error) {
|
||||
if args.TencentCOS == nil {
|
||||
return nil, oss.ErrArgumentInvalid.WithDetail("can't find Tencent COS argument in OSSArgs")
|
||||
}
|
||||
|
||||
err := args.TencentCOS.Validate()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bucket := args.TencentCOS.Bucket
|
||||
region := args.TencentCOS.Region
|
||||
secretID := args.TencentCOS.SecretID
|
||||
secretKey := args.TencentCOS.SecretKey
|
||||
endpoint := args.TencentCOS.Endpoint
|
||||
if endpoint == "" {
|
||||
endpoint = "https://" + bucket + ".cos." + region + ".myqcloud.com"
|
||||
}
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err).WithDetail("url parse failed")
|
||||
}
|
||||
|
||||
b := &cos.BaseURL{BucketURL: u}
|
||||
client := cos.NewClient(b, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: secretID,
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
|
||||
_, err = client.Bucket.Head(context.Background())
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
|
||||
return &TencentCOSStorage{
|
||||
bucket: bucket,
|
||||
region: region,
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *TencentCOSStorage) Save(key string, data []byte) error {
|
||||
_, err := s.client.Object.Put(context.Background(), key, bytes.NewReader(data), nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *TencentCOSStorage) Load(key string) ([]byte, error) {
|
||||
resp, err := s.client.Object.Get(context.Background(), key, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
|
||||
func (s *TencentCOSStorage) Exists(key string) (bool, error) {
|
||||
ok, err := s.client.Object.IsExist(context.Background(), key)
|
||||
if err == nil && ok {
|
||||
return true, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
} else {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TencentCOSStorage) Delete(key string) error {
|
||||
_, err := s.client.Object.Delete(context.Background(), key)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *TencentCOSStorage) List(prefix string) ([]oss.OSSPath, error) {
|
||||
if !strings.HasSuffix(prefix, "/") {
|
||||
prefix = prefix + "/"
|
||||
}
|
||||
|
||||
var keys []oss.OSSPath
|
||||
opt := &cos.BucketGetOptions{
|
||||
Prefix: prefix,
|
||||
Delimiter: "/",
|
||||
}
|
||||
isTruncated := true
|
||||
var marker string
|
||||
for isTruncated {
|
||||
if marker != "" {
|
||||
opt.Marker = marker
|
||||
}
|
||||
|
||||
result, _, err := s.client.Bucket.Get(context.Background(), opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, content := range result.Contents {
|
||||
// remove prefix
|
||||
key := strings.TrimPrefix(content.Key, prefix)
|
||||
// remove leading slash
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, oss.OSSPath{
|
||||
Path: key,
|
||||
IsDir: false,
|
||||
})
|
||||
}
|
||||
|
||||
for _, commonPrefix := range result.CommonPrefixes {
|
||||
if commonPrefix == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.HasSuffix(commonPrefix, "/") {
|
||||
commonPrefix = commonPrefix + "/"
|
||||
}
|
||||
keys = append(keys, oss.OSSPath{
|
||||
Path: commonPrefix,
|
||||
IsDir: true,
|
||||
})
|
||||
|
||||
subKeys, _ := s.List(commonPrefix)
|
||||
if len(subKeys) > 0 {
|
||||
subPrefix := strings.TrimPrefix(commonPrefix, prefix)
|
||||
for i := range subKeys {
|
||||
subKeys[i].Path = subPrefix + subKeys[i].Path
|
||||
}
|
||||
keys = append(keys, subKeys...)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
isTruncated = result.IsTruncated
|
||||
marker = result.NextMarker
|
||||
}
|
||||
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (s *TencentCOSStorage) State(key string) (oss.OSSState, error) {
|
||||
resp, err := s.client.Object.Head(context.Background(), key, nil)
|
||||
if err != nil {
|
||||
return oss.OSSState{}, err
|
||||
}
|
||||
|
||||
contentLength := resp.ContentLength
|
||||
|
||||
lastModified, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
|
||||
if err != nil {
|
||||
lastModified = time.Time{}
|
||||
}
|
||||
|
||||
return oss.OSSState{
|
||||
Size: contentLength,
|
||||
LastModified: lastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *TencentCOSStorage) Type() string {
|
||||
return oss.OSS_TYPE_TENCENT_COS
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package volcenginetos
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
"github.com/volcengine/ve-tos-golang-sdk/v2/tos"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type VolcengineTOSStorage struct {
|
||||
bucket string
|
||||
client *tos.ClientV2
|
||||
}
|
||||
|
||||
func NewVolcengineTOSStorage(args oss.OSSArgs) (oss.OSS, error) {
|
||||
if args.VolcengineTOS == nil {
|
||||
return nil, oss.ErrArgumentInvalid.WithDetail("can't find Volcengine TOS argument in OSSArgs")
|
||||
}
|
||||
|
||||
err := args.VolcengineTOS.Validate()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bucket := args.VolcengineTOS.Bucket
|
||||
accessKey := args.VolcengineTOS.AccessKey
|
||||
secretKey := args.VolcengineTOS.SecretKey
|
||||
endpoint := args.VolcengineTOS.Endpoint
|
||||
region := args.VolcengineTOS.Region
|
||||
|
||||
client, err := tos.NewClientV2(endpoint,
|
||||
tos.WithRegion(region),
|
||||
tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, oss.ErrProviderInit.WithError(err)
|
||||
}
|
||||
return &VolcengineTOSStorage{
|
||||
bucket: bucket,
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *VolcengineTOSStorage) Save(key string, data []byte) error {
|
||||
_, err := s.client.PutObjectV2(context.Background(), &tos.PutObjectV2Input{
|
||||
PutObjectBasicInput: tos.PutObjectBasicInput{
|
||||
Bucket: s.bucket,
|
||||
Key: key,
|
||||
},
|
||||
Content: bytes.NewReader(data),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *VolcengineTOSStorage) Load(key string) ([]byte, error) {
|
||||
resp, err := s.client.GetObjectV2(context.Background(), &tos.GetObjectV2Input{
|
||||
Bucket: s.bucket,
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.ReadAll(resp.Content)
|
||||
}
|
||||
|
||||
func (s *VolcengineTOSStorage) Exists(key string) (bool, error) {
|
||||
_, err := s.client.HeadObjectV2(context.Background(), &tos.HeadObjectV2Input{
|
||||
Bucket: s.bucket,
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
if tosErr, ok := err.(*tos.TosServerError); ok && tosErr.StatusCode == 404 {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *VolcengineTOSStorage) State(key string) (oss.OSSState, error) {
|
||||
resp, err := s.client.HeadObjectV2(context.Background(), &tos.HeadObjectV2Input{
|
||||
Bucket: s.bucket,
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
return oss.OSSState{}, err
|
||||
}
|
||||
return oss.OSSState{
|
||||
Size: resp.ContentLength,
|
||||
LastModified: resp.LastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *VolcengineTOSStorage) List(prefix string) ([]oss.OSSPath, error) {
|
||||
if !strings.HasSuffix(prefix, "/") {
|
||||
prefix = prefix + "/"
|
||||
}
|
||||
var result []oss.OSSPath
|
||||
truncated := true
|
||||
continuationToken := ""
|
||||
for truncated {
|
||||
|
||||
resp, err := s.client.ListObjectsType2(context.Background(), &tos.ListObjectsType2Input{
|
||||
Bucket: s.bucket,
|
||||
Prefix: prefix,
|
||||
MaxKeys: 1000,
|
||||
ContinuationToken: continuationToken,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, obj := range resp.Contents {
|
||||
// remove prefix
|
||||
key := strings.TrimPrefix(obj.Key, prefix)
|
||||
// remove leading slash
|
||||
key = strings.TrimPrefix(key, "/")
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
result = append(result, oss.OSSPath{
|
||||
Path: key,
|
||||
IsDir: false,
|
||||
})
|
||||
}
|
||||
if !resp.IsTruncated {
|
||||
break
|
||||
}
|
||||
truncated = resp.IsTruncated
|
||||
continuationToken = resp.NextContinuationToken
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *VolcengineTOSStorage) Delete(key string) error {
|
||||
_, err := s.client.DeleteObjectV2(context.Background(), &tos.DeleteObjectV2Input{
|
||||
Bucket: s.bucket,
|
||||
Key: key,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *VolcengineTOSStorage) Type() string {
|
||||
return oss.OSS_TYPE_VOLCENGINE_TOS
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/langgenius/dify-cloud-kit/oss"
|
||||
"github.com/langgenius/dify-cloud-kit/oss/factory"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
type testArgsCases struct {
|
||||
vendor string
|
||||
args oss.OSSArgs
|
||||
skip bool
|
||||
}
|
||||
|
||||
var allCases = []testArgsCases{
|
||||
{
|
||||
vendor: "local",
|
||||
args: oss.OSSArgs{
|
||||
Local: &oss.Local{
|
||||
Path: "/tmp/dify-oss-tests",
|
||||
},
|
||||
},
|
||||
skip: false,
|
||||
},
|
||||
{
|
||||
vendor: "s3",
|
||||
args: oss.OSSArgs{
|
||||
S3: &oss.S3{
|
||||
UseAws: true,
|
||||
UsePathStyle: true,
|
||||
UseIamRole: false,
|
||||
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
|
||||
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
|
||||
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
||||
Region: os.Getenv("AWS_S3_REGION"),
|
||||
},
|
||||
},
|
||||
skip: false,
|
||||
},
|
||||
{
|
||||
vendor: "azure",
|
||||
args: oss.OSSArgs{
|
||||
AzureBlob: &oss.AzureBlob{
|
||||
ConnectionString: os.Getenv("AZURE_CONNECTION"),
|
||||
ContainerName: os.Getenv("AZURE_CONTAINER"),
|
||||
},
|
||||
},
|
||||
skip: false,
|
||||
},
|
||||
{
|
||||
vendor: "aliyun",
|
||||
args: oss.OSSArgs{
|
||||
AliyunOSS: &oss.AliyunOSS{
|
||||
Region: os.Getenv("ALIYUN_OSS_REGION"),
|
||||
Endpoint: os.Getenv("ALIYUN_OSS_ENDPOINT"),
|
||||
AccessKey: os.Getenv("ALIYUN_OSS_ACCESS_KEY"),
|
||||
SecretKey: os.Getenv("ALIYUN_OSS_SECRET_KEY"),
|
||||
AuthVersion: os.Getenv("ALIYUN_OSS_AUTH_VERSION"),
|
||||
Path: os.Getenv("ALIYUN_OSS_PATH"),
|
||||
Bucket: os.Getenv("ALIYUN_OSS_BUCKET"),
|
||||
CloudBoxId: os.Getenv("ALIYUN_OSS_CLOUDBOX_ID"),
|
||||
},
|
||||
},
|
||||
skip: false,
|
||||
},
|
||||
{
|
||||
vendor: "tencent",
|
||||
args: oss.OSSArgs{
|
||||
TencentCOS: &oss.TencentCOS{
|
||||
Region: os.Getenv("TENCNET_COS_REGION"),
|
||||
SecretID: os.Getenv("TENCNET_COS_SECRET_ID"),
|
||||
SecretKey: os.Getenv("TENCNET_COS_SECRET_KEY"),
|
||||
Bucket: os.Getenv("TENCNET_COS_BUCKET"),
|
||||
},
|
||||
},
|
||||
skip: false,
|
||||
},
|
||||
{
|
||||
vendor: "gcs",
|
||||
args: oss.OSSArgs{
|
||||
GoogleCloudStorage: &oss.GoogleCloudStorage{
|
||||
Bucket: os.Getenv("GCS_BUCKET"),
|
||||
CredentialsB64: os.Getenv("GCS_CREDENTIALS"),
|
||||
},
|
||||
},
|
||||
skip: false,
|
||||
},
|
||||
{
|
||||
vendor: "huawei",
|
||||
args: oss.OSSArgs{
|
||||
HuaweiOBS: &oss.HuaweiOBS{
|
||||
Bucket: os.Getenv("HUAWEI_OBS_BUCKET"),
|
||||
AccessKey: os.Getenv("HUAWEI_OBS_ACCESS_KEY"),
|
||||
SecretKey: os.Getenv("HUAWEI_OBS_SECRET_KEY"),
|
||||
Server: os.Getenv("HUAWEI_OBS_SERVER"),
|
||||
PathStyle: true,
|
||||
},
|
||||
},
|
||||
skip: false,
|
||||
},
|
||||
{
|
||||
vendor: "volcengine",
|
||||
args: oss.OSSArgs{
|
||||
VolcengineTOS: &oss.VolcengineTOS{
|
||||
Region: os.Getenv("VOLCENGINE_TOS_REGION"),
|
||||
Endpoint: os.Getenv("VOLCENGINE_TOS_ENDPOINT"),
|
||||
AccessKey: os.Getenv("VOLCENGINE_TOS_ACCESS_KEY"),
|
||||
SecretKey: os.Getenv("VOLCENGINE_TOS_SECRET_KEY"),
|
||||
Bucket: os.Getenv("VOLCENGINE_TOS_BUCKET"),
|
||||
},
|
||||
},
|
||||
skip: false,
|
||||
},
|
||||
{
|
||||
vendor: "azure",
|
||||
args: oss.OSSArgs{
|
||||
AzureBlob: &oss.AzureBlob{
|
||||
UseManagedIdentity: true,
|
||||
ContainerName: os.Getenv("AZURE_CONTAINER"),
|
||||
ServiceURL: os.Getenv("AZURE_SERVICE_URL"),
|
||||
},
|
||||
},
|
||||
skip: true,
|
||||
},
|
||||
}
|
||||
|
||||
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
|
||||
func randomString(length int) string {
|
||||
seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
result := make([]byte, length)
|
||||
for i := range result {
|
||||
result[i] = charset[seededRand.Intn(len(charset))]
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
func TestAll(t *testing.T) {
|
||||
prefix := randomString(5)
|
||||
key := fmt.Sprintf("%s/%s", prefix, randomString(10))
|
||||
vendorFilter := os.Getenv("OSS_TEST_VENDOR")
|
||||
dirPlaceholderKey := ""
|
||||
|
||||
size := 1 * 1024 * 1024
|
||||
data := make([]byte, size)
|
||||
|
||||
tested := false
|
||||
for _, c := range allCases {
|
||||
if c.skip {
|
||||
continue
|
||||
}
|
||||
if vendorFilter != "" && c.vendor != vendorFilter {
|
||||
continue
|
||||
}
|
||||
tested = true
|
||||
storage, err := factory.Load(c.vendor, c.args)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
continue
|
||||
}
|
||||
info := fmt.Sprintf("vendor: %s", c.vendor)
|
||||
ossPaths, err := storage.List(prefix)
|
||||
assert.Equal(t, 0, len(ossPaths), info)
|
||||
assert.Nil(t, err, info)
|
||||
|
||||
exist, err := storage.Exists(key)
|
||||
assert.Equal(t, false, exist, info)
|
||||
assert.Nil(t, err, info)
|
||||
|
||||
err = storage.Save(key, data)
|
||||
assert.Nil(t, err, info)
|
||||
|
||||
if c.vendor == "huawei" {
|
||||
dirPlaceholderKey = fmt.Sprintf("%s/langgenius/", prefix)
|
||||
err = storage.Save(dirPlaceholderKey, []byte{})
|
||||
assert.Nil(t, err, info)
|
||||
}
|
||||
|
||||
rdata, err := storage.Load(key)
|
||||
assert.Equal(t, data, rdata, info)
|
||||
assert.Nil(t, err, info)
|
||||
|
||||
ossState, err := storage.State(key)
|
||||
assert.Equal(t, int64(size), ossState.Size, info)
|
||||
assert.Nil(t, err, info)
|
||||
|
||||
exist, err = storage.Exists(key)
|
||||
assert.Equal(t, true, exist, info)
|
||||
assert.Nil(t, err, info)
|
||||
|
||||
ossPaths, err = storage.List(prefix)
|
||||
assert.Equal(t, 1, len(ossPaths), info)
|
||||
assert.Nil(t, err, info)
|
||||
// Verify that List returns relative paths (without prefix)
|
||||
expectedRelativePath := key[len(prefix)+1:] // +1 for the "/" separator
|
||||
if assert.Equal(t, 1, len(ossPaths), info) {
|
||||
assert.Equal(t, expectedRelativePath, ossPaths[0].Path, info+" - List should return relative path")
|
||||
}
|
||||
|
||||
err = storage.Delete(key)
|
||||
assert.Nil(t, err, info)
|
||||
|
||||
if dirPlaceholderKey != "" {
|
||||
err = storage.Delete(dirPlaceholderKey)
|
||||
assert.Nil(t, err, info)
|
||||
dirPlaceholderKey = ""
|
||||
}
|
||||
|
||||
exist, err = storage.Exists(key)
|
||||
assert.Equal(t, false, exist, info)
|
||||
assert.Nil(t, err, info)
|
||||
|
||||
ossPaths, err = storage.List(prefix)
|
||||
assert.Equal(t, 0, len(ossPaths), info)
|
||||
assert.Nil(t, err, info)
|
||||
}
|
||||
|
||||
if vendorFilter != "" && !tested {
|
||||
t.Fatalf("no oss test cases matched OSS_TEST_VENDOR=%q", vendorFilter)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user