diff --git a/README.md b/README.md index e6d0777..ac4fe61 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,109 @@ -# go-cloud-kit -A library and tools for open cloud development in Go. +# Dify Cloud Kit + +Dify Cloud Kit is a unified abstraction library for integrating various cloud object storage services in Go. It simplifies switching between providers and supports local testing and multi-cloud deployments. + +## โœจ Features + +- Supports multiple backends: Local FS, Aliyun OSS, AWS S3, Azure Blob, Tencent COS, Huawei OBS, Google GCS +- Unified and clean interface +- Factory pattern to dynamically load drivers +- Easy to write tests with local and in-memory backends + +## ๐Ÿ“ฆ Installation + +```bash +go get github.com/langgenius/dify-cloud-kit +``` + +## ๐Ÿš€ Quick Start + +```go +import ( + "github.com/langgenius/dify-cloud-kit/oss" + "github.com/langgenius/dify-cloud-kit/oss/factory" +) + +func main() { + store, err := factory.Load("local", oss.OSSArgs{ + Local: &oss.Local{ + Path: "/tmp/files", + }, + }) + if err != nil { + panic(err) + } + + files, _ := store.List("/") + fmt.Println(files) +} +``` + +## ๐Ÿ“ Supported Storage Providers + +| Provider | Module Path | Required Fields | +|--------------|------------------------------|-----------------| +| Local | `oss/local/localfile.go` | `Path` | +| Aliyun OSS | `oss/aliyun/aliyun.go` | `Endpoint`, `AccessKey`, `SecretKey`, `Bucket` | +| AWS S3 | `oss/s3/s3.go` | `Region`, `AccessKey`, `SecretKey`, `Bucket` | +| Azure Blob | `oss/azureblob/blob.go` | `AccountName`, `AccountKey`, `Container` | +| Google GCS | `oss/gcsblob/gcs.go` | `CredentialsJSON`, `Bucket` | +| Tencent COS | `oss/tencentcos/cos.go` | `SecretId`, `SecretKey`, `Bucket`, `Region` | +| Huawei OBS | `oss/huanweiobs/obs.go` | `AK`, `SK`, `Endpoint`, `Bucket` | + +## ๐Ÿ—๏ธ Usage with Factory + +You can dynamically load a storage backend using the factory: + +```go +store, err := factory.Load("s3", oss.OSSArgs{ + S3: &oss.S3{ + Region: "us-west-2", + AccessKey: "AKIA...", + SecretKey: "SECRET...", + Bucket: "my-bucket", + }, +}) +``` + +## ๐Ÿงช Testing + +Unit tests are located in `tests/oss/oss_test.go`. + +### Environment Variables + +Some providers require credentials to be passed via environment variables. Set them as needed: + +```bash +export OSS_AWS_ACCESS_KEY=your-access-key +export OSS_AWS_SECRET_KEY=your-secret-key +export OSS_AWS_REGION=your-region +export OSS_S3_BUCKET=test-bucket + +export OSS_ALIYUN_ENDPOINT=your-endpoint +export OSS_ALIYUN_ACCESS_KEY=your-access-key +export OSS_ALIYUN_SECRET_KEY=your-secret-key +export OSS_ALIYUN_BUCKET=test-bucket + +``` + +### Run Tests + +```bash +go test ./... +``` + +## ๐Ÿ“„ License + +This project is licensed under the [Apache 2.0 License](LICENSE). + +## NOTICE +Some parts of the code in this project originate from [dify-plugin-daemon](https://github.com/langgenius/dify-plugin-daemon) + +|Provider | Author | PR | +|---|---|---| +|Aliyun OSS|[bravomark](https://github.com/bravomark)|https://github.com/langgenius/dify-plugin-daemon/pull/261 | +|Azure Blob|[techan](https://github.com/te-chan)|https://github.com/langgenius/dify-plugin-daemon/pull/172| +|Google GCS|[Hironori Yamamoto](https://github.com/hiro-o918)|https://github.com/langgenius/dify-plugin-daemon/pull/237| +|Local|[lengyhua](https://github.com/lengyhua)|https://github.com/langgenius/dify-plugin-daemon/pull/157| +|AWS S3|[Yeuoly](https://github.com/Yeuoly)|https://github.com/langgenius/dify-plugin-daemon/commit/9ad9d7d4de1d123956ab07955e541bc4053e5170| +|Tencent COS|[quicksand](https://github.com/quicksandznzn)|https://github.com/langgenius/dify-plugin-daemon/pull/97| diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..738be47 --- /dev/null +++ b/go.mod @@ -0,0 +1,89 @@ +module github.com/langgenius/dify-cloud-kit + +go 1.23.3 + +require ( + cloud.google.com/go/storage v1.54.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible + github.com/aws/aws-sdk-go-v2 v1.36.3 + github.com/aws/aws-sdk-go-v2/config v1.29.14 + github.com/aws/aws-sdk-go-v2/credentials v1.17.67 + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4 + github.com/aws/smithy-go v1.22.2 + github.com/stretchr/testify v1.10.0 + github.com/tencentyun/cos-go-sdk-v5 v0.7.65 +) + +require ( + cel.dev/expr v0.20.0 // indirect + cloud.google.com/go v0.121.0 // indirect + cloud.google.com/go/auth v0.16.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect + cloud.google.com/go/iam v1.5.2 // indirect + cloud.google.com/go/monitoring v1.24.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clbanning/mxj v1.8.4 // indirect + github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/go-querystring v1.0.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect + github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/mozillazg/go-httpheader v0.2.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.14.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect + golang.org/x/time v0.11.0 // indirect + google.golang.org/api v0.232.0 // indirect + google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2 // indirect + google.golang.org/grpc v1.72.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..05b9b8a --- /dev/null +++ b/go.sum @@ -0,0 +1,210 @@ +cel.dev/expr v0.20.0 h1:OunBvVCfvpWlt4dN7zg3FM6TDkzOePe1+foGJ9AXeeI= +cel.dev/expr v0.20.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +cloud.google.com/go v0.121.0 h1:pgfwva8nGw7vivjZiRfrmglGWiCJBP+0OmDpenG/Fwg= +cloud.google.com/go v0.121.0/go.mod h1:rS7Kytwheu/y9buoDmu5EIpMMCI4Mb8ND4aeN4Vwj7Q= +cloud.google.com/go/auth v0.16.1 h1:XrXauHMd30LhQYVRHLGvJiYeczweKQXZxsTbV9TiguU= +cloud.google.com/go/auth v0.16.1/go.mod h1:1howDHJ5IETh/LwYs3ZxvlkXF48aSqqJUM+5o02dNOI= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= +cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= +cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= +cloud.google.com/go/monitoring v1.24.0 h1:csSKiCJ+WVRgNkRzzz3BPoGjFhjPY23ZTcaenToJxMM= +cloud.google.com/go/monitoring v1.24.0/go.mod h1:Bd1PRK5bmQBQNnuGwHBfUamAV1ys9049oEPHnn4pcsc= +cloud.google.com/go/storage v1.54.0 h1:Du3XEyliAiftfyW0bwfdppm2MMLdpVAfiIg4T2nAI+0= +cloud.google.com/go/storage v1.54.0/go.mod h1:hIi9Boe8cHxTyaeqh7KMMwKg088VblFK46C2x/BWaZE= +cloud.google.com/go/trace v1.11.3 h1:c+I4YFjxRQjvAhRmSsmjpASUKq88chOX854ied0K/pE= +cloud.google.com/go/trace v1.11.3/go.mod h1:pt7zCYiDSQjC9Y2oqCsh9jF4GStB/hmjrYLsxRR27q8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= +github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= +github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= +github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 h1:BCG7DCXEXpNCcpwCxg1oi9pkJWH2+eZzTn9MY56MbVw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4 h1:4yxno6bNHkekkfqG/a1nz/gC2gBwhJSojV1+oTE7K+4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4/go.mod h1:qbn305Je/IofWBJ4bJz/Q7pDEtnnoInw/dGt71v6rHE= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= +github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= +github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 h1:Om6kYQYDUk5wWbT0t0q6pvyM49i9XZAv9dDrkDA7gjk= +github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= +github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible h1:yNjwdvn9fwuN6Ouxr0xHM0cVu03YMUWUyFmu2van/Yc= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= +github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0= +github.com/tencentyun/cos-go-sdk-v5 v0.7.65 h1:+WBbfwThfZSbxpf1Dw6fyMwyzVtWBBExqfDJ5giiR2s= +github.com/tencentyun/cos-go-sdk-v5 v0.7.65/go.mod h1:8+hG+mQMuRP/OIS9d83syAvXvrMj9HhkND6Q1fLghw0= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/detectors/gcp v1.35.0 h1:bGvFt68+KTiAKFlacHW6AhA56GF2rS0bdD3aJYEnmzA= +go.opentelemetry.io/contrib/detectors/gcp v1.35.0/go.mod h1:qGWP8/+ILwMRIUf9uIVLloR1uo5ZYAslM4O6OqUi1DA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +google.golang.org/api v0.232.0 h1:qGnmaIMf7KcuwHOlF3mERVzChloDYwRfOJOrHt8YC3I= +google.golang.org/api v0.232.0/go.mod h1:p9QCfBWZk1IJETUdbTKloR5ToFdKbYh2fkjsUL6vNoY= +google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE= +google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE= +google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 h1:vPV0tzlsK6EzEDHNNH5sa7Hs9bd7iXR7B1tSiPepkV0= +google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:pKLAc5OolXC3ViWGI62vvC0n10CpwAtRcTNCFwTKBEw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2 h1:IqsN8hx+lWLqlN+Sc3DoMy/watjofWiU8sRFgQ8fhKM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= +google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/oss/aliyun/aliyun.go b/oss/aliyun/aliyun.go new file mode 100644 index 0000000..1a1bf8e --- /dev/null +++ b/oss/aliyun/aliyun.go @@ -0,0 +1,203 @@ +package aliyun + +import ( + "bytes" + "fmt" + "io" + "path" + "strings" + "time" + + "github.com/aliyun/aliyun-oss-go-sdk/oss" + difyoss "github.com/langgenius/dify-cloud-kit/oss" +) + +type AliyunOSSStorage struct { + client *oss.Client + bucket *oss.Bucket + path string +} + +func NewAliyunOSSStorage(args difyoss.OSSArgs) (difyoss.OSS, error) { + var err error + if args.AliyunOSS == nil { + return nil, difyoss.ErrArgumentInvalid.WithDetail("can't find Aiyun OSS argument in OSSArgs") + } + + err = args.AliyunOSS.Validate() + if err != nil { + return nil, err + } + + region := args.AliyunOSS.Region + endpoint := args.AliyunOSS.Endpoint + accessKeyID := args.AliyunOSS.AccessKey + accessKeySecret := args.AliyunOSS.SecretKey + authVersion := args.AliyunOSS.AuthVersion + path := args.AliyunOSS.Path + bucketName := args.AliyunOSS.Bucket + + // options + var options []oss.ClientOption + + // set region (required for v4) + if region != "" { + options = append(options, oss.Region(region)) + } + + // set auth-version + if authVersion == "v1" { + options = append(options, oss.AuthVersion(oss.AuthV1)) + } else if authVersion == "v4" { + options = append(options, oss.AuthVersion(oss.AuthV4)) + } else { + // default use v4 + options = append(options, oss.AuthVersion(oss.AuthV4)) + } + + // create client + var client *oss.Client + + client, err = oss.New(endpoint, accessKeyID, accessKeySecret, options...) + + if err != nil { + return nil, difyoss.ErrProviderInit.WithError(err).WithDetail("failed to create Aliyun OSS client") + } + + // get specified bucket + bucket, err := client.Bucket(bucketName) + if err != nil { + return nil, difyoss.ErrProviderInit.WithError(err).WithDetail(fmt.Sprintf("failed to get bucket %s", bucketName)) + } + + // normalize path: remove leading slash, ensure trailing slash + path = strings.TrimPrefix(path, "/") + if path != "" && !strings.HasSuffix(path, "/") { + path = path + "/" + } + + return &AliyunOSSStorage{ + client: client, + bucket: bucket, + path: path, + }, nil +} + +// combine full object path +func (s *AliyunOSSStorage) fullPath(key string) string { + return path.Join(s.path, key) +} + +func (s *AliyunOSSStorage) Save(key string, data []byte) error { + fullPath := s.fullPath(key) + return s.bucket.PutObject(fullPath, bytes.NewReader(data)) +} + +func (s *AliyunOSSStorage) Load(key string) ([]byte, error) { + fullPath := s.fullPath(key) + object, err := s.bucket.GetObject(fullPath) + if err != nil { + return nil, err + } + // Ensure object is closed after reading + defer object.Close() + + data, err := io.ReadAll(object) + if err != nil { + return nil, err + } + return data, nil +} + +func (s *AliyunOSSStorage) Exists(key string) (bool, error) { + fullPath := s.fullPath(key) + return s.bucket.IsObjectExist(fullPath) +} + +func (s *AliyunOSSStorage) State(key string) (difyoss.OSSState, error) { + fullPath := s.fullPath(key) + meta, err := s.bucket.GetObjectMeta(fullPath) + if err != nil { + return difyoss.OSSState{}, err + } + + // Get content length + size := int64(0) + contentLength := meta.Get("Content-Length") + if contentLength != "" { + _, err := fmt.Sscanf(contentLength, "%d", &size) + if err != nil { + // Return zero size if parsing fails + size = 0 + } + } + + // Get last modified time + lastModified := time.Time{} + lastModifiedStr := meta.Get("Last-Modified") + if lastModifiedStr != "" { + lastModified, err = time.Parse(time.RFC1123, lastModifiedStr) + if err != nil { + // Return zero time if parsing fails + lastModified = time.Time{} + } + } + + return difyoss.OSSState{ + Size: size, + LastModified: lastModified, + }, nil +} + +func (s *AliyunOSSStorage) List(prefix string) ([]difyoss.OSSPath, error) { + // combine given prefix with path + fullPrefix := s.fullPath(prefix) + + // Ensure the prefix ends with a slash for directories + if !strings.HasSuffix(fullPrefix, "/") { + fullPrefix = fullPrefix + "/" + } + + var keys []difyoss.OSSPath + marker := "" + for { + lsRes, err := s.bucket.ListObjects(oss.Marker(marker), oss.Prefix(fullPrefix)) + if err != nil { + return nil, fmt.Errorf("failed to list objects in Aliyun OSS: %w", err) + } + + for _, object := range lsRes.Objects { + if object.Key == fullPrefix { + continue + } + // remove path and prefix from full path, only keep relative path + key := strings.TrimPrefix(object.Key, fullPrefix) + // Skip empty keys and directories (keys ending with /) + if key == "" || strings.HasSuffix(key, "/") { + continue + } + keys = append(keys, difyoss.OSSPath{ + Path: key, + IsDir: false, + }) + } + + // Check if there are more results + if lsRes.IsTruncated { + marker = lsRes.NextMarker + } else { + break + } + } + + return keys, nil +} + +func (s *AliyunOSSStorage) Delete(key string) error { + fullPath := s.fullPath(key) + return s.bucket.DeleteObject(fullPath) +} + +func (s *AliyunOSSStorage) Type() string { + return difyoss.OSS_TYPE_ALIYUN_OSS +} diff --git a/oss/azureblob/blob.go b/oss/azureblob/blob.go new file mode 100644 index 0000000..c0138ac --- /dev/null +++ b/oss/azureblob/blob.go @@ -0,0 +1,131 @@ +package azureblob + +import ( + "bytes" + "context" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/langgenius/dify-cloud-kit/oss" +) + +type AzureBlobStorage struct { + client *azblob.Client + containerName string +} + +func NewAzureBlobStorage(args oss.OSSArgs) (oss.OSS, error) { + if args.AzureBlob == nil { + return nil, oss.ErrArgumentInvalid.WithDetail("can't find Azure Blob argument in OSSArgs") + } + err := args.AzureBlob.Validate() + if err != nil { + return nil, err + } + connectionString := args.AzureBlob.ConnectionString + containerName := args.AzureBlob.ContainerName + client, err := azblob.NewClientFromConnectionString(connectionString, nil) + if err != nil { + return nil, oss.ErrProviderInit.WithError(err) + } + + return &AzureBlobStorage{ + client: client, + containerName: containerName, + }, nil +} + +func (a *AzureBlobStorage) Save(key string, data []byte) error { + _, err := a.client.UploadBuffer(context.TODO(), a.containerName, key, data, nil) + return err +} + +func (a *AzureBlobStorage) Load(key string) ([]byte, error) { + get, err := a.client.DownloadStream(context.TODO(), a.containerName, key, nil) + if err != nil { + return nil, err + } + + downloadedData := bytes.Buffer{} + retryReader := get.NewRetryReader(context.TODO(), &azblob.RetryReaderOptions{}) + _, err = downloadedData.ReadFrom(retryReader) + if err != nil { + return nil, err + } + + err = retryReader.Close() + if err != nil { + return nil, err + } + + return downloadedData.Bytes(), nil +} + +func (a *AzureBlobStorage) Exists(key string) (bool, error) { + blobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlobClient(key) + _, err := blobClient.GetProperties(context.TODO(), nil) + + if err != nil { + if strings.Contains(err.Error(), "404") { + return false, nil + } + return false, err + } + + return true, nil +} + +func (a *AzureBlobStorage) State(key string) (oss.OSSState, error) { + blobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlobClient(key) + props, err := blobClient.GetProperties(context.TODO(), nil) + + if err != nil { + return oss.OSSState{}, err + } + + return oss.OSSState{ + Size: *props.ContentLength, + LastModified: *props.LastModified, + }, nil +} + +func (a *AzureBlobStorage) List(prefix string) ([]oss.OSSPath, error) { + // append a slash to the prefix if it doesn't end with one + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + + pager := a.client.NewListBlobsFlatPager(a.containerName, &azblob.ListBlobsFlatOptions{ + Prefix: &prefix, + }) + + paths := make([]oss.OSSPath, 0) + for pager.More() { + page, err := pager.NextPage(context.TODO()) + if err != nil { + return nil, err + } + + for _, blob := range page.Segment.BlobItems { + // remove prefix + key := strings.TrimPrefix(*blob.Name, prefix) + // remove leading slash + key = strings.TrimPrefix(key, "/") + paths = append(paths, oss.OSSPath{ + Path: key, + IsDir: false, + }) + } + } + + return paths, nil +} + +func (a *AzureBlobStorage) Delete(key string) error { + _, err := a.client.DeleteBlob(context.TODO(), a.containerName, key, nil) + return err +} + +func (a *AzureBlobStorage) Type() string { + return oss.OSS_TYPE_AZURE_BLOB +} diff --git a/oss/errors.go b/oss/errors.go new file mode 100644 index 0000000..efff338 --- /dev/null +++ b/oss/errors.go @@ -0,0 +1,39 @@ +package oss + +import "fmt" + +var ( + ErrProviderNotFound = NewCloudKitError("provider not found", "") + ErrArgumentInvalid = NewCloudKitError("argument invalid", "") + ErrProviderInit = NewCloudKitError("provider init error", "") +) + +type CloudKitError struct { + Reason string + Detail string + Err error +} + +func NewCloudKitError(reason string, detail string) *CloudKitError { + return &CloudKitError{ + Reason: reason, + Detail: detail, + } +} + +func (c *CloudKitError) Error() string { + if c.Detail != "" { + return fmt.Sprintf("reason: %s; detail: %s; error: %v", c.Reason, c.Detail, c.Err) + } + return fmt.Sprintf("reason: %s; error: %v", c.Reason, c.Err) +} + +func (c *CloudKitError) WithDetail(detail string) *CloudKitError { + c.Detail = detail + return c +} + +func (c *CloudKitError) WithError(err error) *CloudKitError { + c.Err = err + return c +} diff --git a/oss/factory/factory.go b/oss/factory/factory.go new file mode 100644 index 0000000..4d37b95 --- /dev/null +++ b/oss/factory/factory.go @@ -0,0 +1,49 @@ +package factory + +import ( + "fmt" + + "github.com/langgenius/dify-cloud-kit/oss" + "github.com/langgenius/dify-cloud-kit/oss/aliyun" + "github.com/langgenius/dify-cloud-kit/oss/azureblob" + "github.com/langgenius/dify-cloud-kit/oss/gcsblob" + "github.com/langgenius/dify-cloud-kit/oss/huanweiobs" + "github.com/langgenius/dify-cloud-kit/oss/local" + "github.com/langgenius/dify-cloud-kit/oss/s3" + "github.com/langgenius/dify-cloud-kit/oss/tencentcos" +) + +var OSSFactory = map[string]func(oss.OSSArgs) (oss.OSS, error){ + "local": local.NewLocalStorage, + "local_file": local.NewLocalStorage, + + "s3": s3.NewS3Storage, + "aws_s3": s3.NewS3Storage, + + "azure": azureblob.NewAzureBlobStorage, + "azure_blob": azureblob.NewAzureBlobStorage, + + "aliyun": aliyun.NewAliyunOSSStorage, + "aliyun-oss": aliyun.NewAliyunOSSStorage, + "aliyun_oss": aliyun.NewAliyunOSSStorage, + + "tencent": tencentcos.NewTencentCOSStorage, + "tencent_cos": tencentcos.NewTencentCOSStorage, + "tencent-cos": tencentcos.NewTencentCOSStorage, + + "gcs": gcsblob.NewGoogleCloudStorage, + "google-storage": gcsblob.NewGoogleCloudStorage, + + "huawei": huanweiobs.NewHuaweiOBSStorage, + "huawei-obs": huanweiobs.NewHuaweiOBSStorage, + "huawei_obs": huanweiobs.NewHuaweiOBSStorage, +} + +func Load(name string, args oss.OSSArgs) (oss.OSS, error) { + f, ok := OSSFactory[name] + if !ok { + msg := fmt.Sprintf("[ %s ] is not in the provider list", name) + return nil, oss.ErrProviderNotFound.WithDetail(msg) + } + return f(args) +} diff --git a/oss/gcsblob/gcs.go b/oss/gcsblob/gcs.go new file mode 100644 index 0000000..919381d --- /dev/null +++ b/oss/gcsblob/gcs.go @@ -0,0 +1,143 @@ +package gcsblob + +import ( + "context" + "encoding/base64" + "errors" + "io" + "strings" + + "cloud.google.com/go/storage" + "github.com/langgenius/dify-cloud-kit/oss" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +type GoogleCloudStorage struct { + bucket string + client *storage.Client +} + +func NewGoogleCloudStorage(args oss.OSSArgs) (oss.OSS, error) { + if args.GoogleCloudStorage == nil { + return nil, oss.ErrArgumentInvalid.WithDetail("can't find Google Cloud Storage argument in OSSArgs") + } + err := args.GoogleCloudStorage.Validate() + if err != nil { + return nil, err + } + ctx := context.Background() + bucket := args.GoogleCloudStorage.Bucket + credentialsB64 := args.GoogleCloudStorage.CredentialsB64 + credentials, err := base64.StdEncoding.DecodeString(credentialsB64) + if err != nil { + return nil, oss.ErrProviderInit.WithError(err).WithDetail("credentials must be a base64 encoded string") + } + client, err := storage.NewClient(ctx, option.WithCredentialsJSON(credentials)) + if err != nil { + return nil, oss.ErrProviderInit.WithError(err) + } + return &GoogleCloudStorage{ + bucket: bucket, + client: client, + }, nil +} + +func (g *GoogleCloudStorage) Save(key string, data []byte) error { + ctx := context.Background() + obj := g.client.Bucket(g.bucket).Object(key) + obj = obj.If(storage.Conditions{DoesNotExist: true}) + + wc := obj.NewWriter(ctx) + if _, err := wc.Write(data); err != nil { + return err + } + return wc.Close() +} + +func (g *GoogleCloudStorage) Load(key string) ([]byte, error) { + rc, err := g.client.Bucket(g.bucket).Object(key).NewReader(context.Background()) + if err != nil { + return nil, err + } + defer rc.Close() + + data, err := io.ReadAll(rc) + if err != nil { + return nil, err + } + return data, nil +} + +func (g *GoogleCloudStorage) Exists(key string) (bool, error) { + obj := g.client.Bucket(g.bucket).Object(key) + + _, err := obj.Attrs(context.Background()) + if err != nil { + if errors.Is(err, storage.ErrObjectNotExist) { + return false, nil + } + return false, err + } + return true, nil +} + +func (g *GoogleCloudStorage) State(key string) (oss.OSSState, error) { + obj := g.client.Bucket(g.bucket).Object(key) + + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return oss.OSSState{}, err + } + return oss.OSSState{ + Size: attrs.Size, + LastModified: attrs.Updated, + }, nil +} + +func (g *GoogleCloudStorage) List(prefix string) ([]oss.OSSPath, error) { + ctx := context.Background() + it := g.client.Bucket(g.bucket).Objects(ctx, &storage.Query{ + Prefix: prefix, + }) + res := []oss.OSSPath{} + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + if attrs.Name == prefix { + continue + } + + key := strings.TrimPrefix(attrs.Name, prefix) + key = strings.TrimPrefix(key, "/") + + res = append(res, oss.OSSPath{ + Path: attrs.Name, + IsDir: false, + }) + + } + return res, nil +} + +func (g *GoogleCloudStorage) Delete(key string) error { + ctx := context.Background() + obj := g.client.Bucket(g.bucket).Object(key) + attrs, err := obj.Attrs(ctx) + if err != nil { + return err + } + + obj = obj.If(storage.Conditions{GenerationMatch: attrs.Generation}) + + return obj.Delete(ctx) +} + +func (g *GoogleCloudStorage) Type() string { + return oss.OSS_TYPE_GCS +} diff --git a/oss/huanweiobs/obs.go b/oss/huanweiobs/obs.go new file mode 100644 index 0000000..808a8c2 --- /dev/null +++ b/oss/huanweiobs/obs.go @@ -0,0 +1,165 @@ +package huanweiobs + +import ( + "io" + "os" + "strings" + "time" + + "math/rand" + + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" + "github.com/langgenius/dify-cloud-kit/oss" +) + +type HuaweiOBSStorage struct { + bucket string + client *obs.ObsClient +} + +func NewHuaweiOBSStorage(args oss.OSSArgs) (oss.OSS, error) { + if args.HuaweiOBS == nil { + return nil, oss.ErrArgumentInvalid.WithDetail("can't find Huawei OBS argument in OSSArgs") + } + + err := args.HuaweiOBS.Validate() + if err != nil { + return nil, err + } + ak := args.HuaweiOBS.AccessKey + sk := args.HuaweiOBS.SecretKey + endpoint := args.HuaweiOBS.Server + bucket := args.HuaweiOBS.Bucket + client, err := obs.New(ak, sk, endpoint) + if err != nil { + return nil, oss.ErrProviderInit.WithError(err) + } + + return &HuaweiOBSStorage{ + bucket: bucket, + client: client, + }, nil +} + +func (h *HuaweiOBSStorage) Save(key string, data []byte) error { + tmpFilename := randomString(5) + file, err := os.CreateTemp("/tmp", tmpFilename) + if err != nil { + return err + } + defer os.Remove(file.Name()) + + _, err = file.Write(data) + if err != nil { + return err + } + + _, err = h.client.PutFile(&obs.PutFileInput{ + PutObjectBasicInput: obs.PutObjectBasicInput{ + ObjectOperationInput: obs.ObjectOperationInput{ + Bucket: h.bucket, + Key: key, + }, + }, + SourceFile: file.Name(), + }) + return err +} + +func (h *HuaweiOBSStorage) Load(key string) ([]byte, error) { + output, err := h.client.GetObject(&obs.GetObjectInput{ + GetObjectMetadataInput: obs.GetObjectMetadataInput{ + Bucket: h.bucket, + Key: key, + }, + }) + if err != nil { + return nil, err + } + defer output.Body.Close() + + return io.ReadAll(output.Body) +} + +func (h *HuaweiOBSStorage) Exists(key string) (bool, error) { + _, err := h.client.HeadObject(&obs.HeadObjectInput{ + Bucket: h.bucket, + Key: key, + }) + + if err == nil { + return true, nil + } + + if obsErr, ok := err.(obs.ObsError); ok && obsErr.StatusCode == 404 { + return false, nil + } + return false, err +} + +func (h *HuaweiOBSStorage) State(key string) (oss.OSSState, error) { + output, err := h.client.GetAttribute(&obs.GetAttributeInput{ + GetObjectMetadataInput: obs.GetObjectMetadataInput{ + Bucket: h.bucket, + Key: key, + }, + }) + if err != nil { + return oss.OSSState{}, err + } + return oss.OSSState{ + Size: output.ContentLength, + LastModified: output.LastModified, + }, nil +} + +func (h *HuaweiOBSStorage) List(prefix string) ([]oss.OSSPath, error) { + output, err := h.client.ListObjects(&obs.ListObjectsInput{ + Bucket: h.bucket, + ListObjsInput: obs.ListObjsInput{ + Prefix: prefix, + }, + }) + if err != nil { + return nil, err + } + paths := []oss.OSSPath{} + for _, v := range output.Contents { + key := strings.TrimPrefix(v.Key, prefix) + key = strings.TrimPrefix(key, "/") + + if key == "" { + continue + } + paths = append(paths, oss.OSSPath{ + Path: v.Key, + IsDir: func() bool { + return strings.HasSuffix(v.Key, "/") + }(), + }) + } + return paths, nil +} + +func (h *HuaweiOBSStorage) Delete(key string) error { + _, err := h.client.DeleteObject(&obs.DeleteObjectInput{ + Bucket: h.bucket, + Key: key, + }) + return err +} + +func (h *HuaweiOBSStorage) Type() string { + return oss.OSS_TYPE_HUAWEI_OBS +} + +const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +func randomString(length int) string { + seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) + result := make([]byte, length) + for i := range result { + result[i] = charset[seededRand.Intn(len(charset))] + } + return string(result) +} diff --git a/oss/local/localfile.go b/oss/local/localfile.go new file mode 100644 index 0000000..aebe0fb --- /dev/null +++ b/oss/local/localfile.go @@ -0,0 +1,111 @@ +package local + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/langgenius/dify-cloud-kit/oss" +) + +type LocalStorage struct { + root string +} + +func NewLocalStorage(args oss.OSSArgs) (oss.OSS, error) { + if args.Local == nil { + return nil, oss.ErrArgumentInvalid.WithDetail("can't find Local argument in OSSArgs") + } + err := args.Local.Validate() + if err != nil { + return nil, err + } + root := args.Local.Path + if err := os.MkdirAll(root, 0755); err != nil { + return nil, oss.ErrProviderInit.WithError(err).WithDetail("failed to create storage path") + } + + return &LocalStorage{root: root}, nil +} + +func (l *LocalStorage) Save(key string, data []byte) error { + path := filepath.Join(l.root, key) + filePath := filepath.Dir(path) + if err := os.MkdirAll(filePath, 0o755); err != nil { + return err + } + + return os.WriteFile(path, data, 0o644) +} + +func (l *LocalStorage) Load(key string) ([]byte, error) { + path := filepath.Join(l.root, key) + + return os.ReadFile(path) +} + +func (l *LocalStorage) Exists(key string) (bool, error) { + path := filepath.Join(l.root, key) + + _, err := os.Stat(path) + return err == nil, nil +} + +func (l *LocalStorage) State(key string) (oss.OSSState, error) { + path := filepath.Join(l.root, key) + + info, err := os.Stat(path) + if err != nil { + return oss.OSSState{}, err + } + + return oss.OSSState{Size: info.Size(), LastModified: info.ModTime()}, nil +} + +func (l *LocalStorage) List(prefix string) ([]oss.OSSPath, error) { + paths := make([]oss.OSSPath, 0) + // check if the patch exists + exists, err := l.Exists(prefix) + if err != nil { + return nil, err + } + if !exists { + return paths, nil + } + prefix = filepath.Join(l.root, prefix) + + err = filepath.WalkDir(prefix, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + // remove prefix + path = strings.TrimPrefix(path, prefix) + if path == "" { + return nil + } + // remove leading slash + path = strings.TrimPrefix(path, "/") + paths = append(paths, oss.OSSPath{ + Path: path, + IsDir: d.IsDir(), + }) + return nil + }) + + if err != nil { + return nil, err + } + + return paths, nil +} + +func (l *LocalStorage) Delete(key string) error { + path := filepath.Join(l.root, key) + + return os.RemoveAll(path) +} + +func (l *LocalStorage) Type() string { + return oss.OSS_TYPE_LOCAL +} diff --git a/oss/local/localfile_test.go b/oss/local/localfile_test.go new file mode 100644 index 0000000..469c3dc --- /dev/null +++ b/oss/local/localfile_test.go @@ -0,0 +1 @@ +package local diff --git a/oss/oss.go b/oss/oss.go new file mode 100644 index 0000000..e553b45 --- /dev/null +++ b/oss/oss.go @@ -0,0 +1,160 @@ +package oss + +import ( + "fmt" + "time" +) + +// OSS supports different types of object storage services +// such as local file system, AWS S3, and Tencent COS. +// The interface defines methods for saving, loading, checking existence, +const ( + OSS_TYPE_LOCAL = "local" + OSS_TYPE_S3 = "aws_s3" + OSS_TYPE_TENCENT_COS = "tencent_cos" + OSS_TYPE_AZURE_BLOB = "azure_blob" + OSS_TYPE_GCS = "gcs" + OSS_TYPE_ALIYUN_OSS = "aliyun_oss" + OSS_TYPE_HUAWEI_OBS = "huawei_obs" +) + +type OSSState struct { + Size int64 + LastModified time.Time +} + +type OSSPath struct { + Path string + IsDir bool +} + +type OSS interface { + // Save saves data into path key + Save(key string, data []byte) error + // Load loads data from path key + Load(key string) ([]byte, error) + // Exists checks if the data exists in the path key + Exists(key string) (bool, error) + // State gets the state of the data in the path key + State(key string) (OSSState, error) + // List lists all the data with the given prefix, and all the paths are absolute paths + List(prefix string) ([]OSSPath, error) + // Delete deletes the data in the path key + Delete(key string) error + // Type returns the type of the storage + // For example: local, aws_s3, tencent_cos + Type() string +} + +type OSSArgs struct { + S3 *S3 + Local *Local + AzureBlob *AzureBlob + AliyunOSS *AliyunOSS + TencentCOS *TencentCOS + GoogleCloudStorage *GoogleCloudStorage + HuaweiOBS *HuaweiOBS +} + +type S3 struct { + UseAws bool + Endpoint string + UsePathStyle bool + AccessKey string + SecretKey string + Bucket string + Region string +} + +func (s *S3) Validate() error { + if s.Bucket == "" || s.Region == "" { + msg := fmt.Sprintf("bucket and region cannot be empty.") + return ErrArgumentInvalid.WithDetail(msg) + } + return nil +} + +type AzureBlob struct { + ConnectionString string + ContainerName string +} + +func (a *AzureBlob) Validate() error { + if a.ConnectionString == "" || a.ContainerName == "" { + msg := fmt.Sprintf("connectorString and containerName cannot be empty.") + return ErrArgumentInvalid.WithDetail(msg) + } + return nil +} + +type Local struct { + Path string +} + +func (l *Local) Validate() error { + if l.Path == "" { + return ErrArgumentInvalid.WithDetail("path cannot be empty") + } + return nil +} + +type AliyunOSS struct { + Region string + Endpoint string + AccessKey string + SecretKey string + AuthVersion string + Path string + Bucket string +} + +func (a *AliyunOSS) Validate() error { + if a.Bucket == "" || a.SecretKey == "" || a.AccessKey == "" || a.Endpoint == "" { + msg := fmt.Sprintf("bucket, accesskKey, secretKey, endpoint cannot be empty.") + return ErrArgumentInvalid.WithDetail(msg) + } + return nil +} + +type TencentCOS struct { + Region string + SecretID string + SecretKey string + Bucket string +} + +func (t *TencentCOS) Validate() error { + if t.Bucket == "" || t.Region == "" || t.SecretID == "" || t.SecretKey == "" { + msg := fmt.Sprintf("bucket, region, secretKey, secretID cannot be empty.") + return ErrArgumentInvalid.WithDetail(msg) + } + return nil +} + +type GoogleCloudStorage struct { + Bucket string + CredentialsB64 string +} + +func (g *GoogleCloudStorage) Validate() error { + if g.Bucket == "" || g.CredentialsB64 == "" { + msg := fmt.Sprintf("bucket and credentials cannot be empty.") + return ErrArgumentInvalid.WithDetail(msg) + } + return nil +} + +type HuaweiOBS struct { + Bucket string + AccessKey string + SecretKey string + Server string +} + +func (h *HuaweiOBS) Validate() error { + if h.Bucket == "" || h.AccessKey == "" || h.SecretKey == "" || h.Server == "" { + msg := fmt.Sprintf("bucket, accesskKey, secretKey, server cannot be empty.") + return ErrArgumentInvalid.WithDetail(msg) + } + return nil +} diff --git a/oss/s3/s3.go b/oss/s3/s3.go new file mode 100644 index 0000000..96a1779 --- /dev/null +++ b/oss/s3/s3.go @@ -0,0 +1,206 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "github.com/langgenius/dify-cloud-kit/oss" +) + +type S3Storage struct { + bucket string + client *s3.Client +} + +func NewS3Storage(args oss.OSSArgs) (oss.OSS, error) { + var err error + if args.S3 == nil { + return nil, oss.ErrArgumentInvalid.WithDetail("can't find s3 argument in OSSArgs") + } + err = args.S3.Validate() + if err != nil { + return nil, err + } + useAws := args.S3.UseAws + ak := args.S3.AccessKey + sk := args.S3.SecretKey + region := args.S3.Region + endpoint := args.S3.Endpoint + usePathStyle := args.S3.UsePathStyle + bucket := args.S3.Bucket + + var cfg aws.Config + var client *s3.Client + + if useAws { + if ak == "" && sk == "" { + cfg, err = config.LoadDefaultConfig( + context.TODO(), + config.WithRegion(region), + ) + } else { + cfg, err = config.LoadDefaultConfig( + context.TODO(), + config.WithRegion(region), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + ak, + sk, + "", + )), + ) + } + if err != nil { + return nil, oss.ErrProviderInit.WithError(err) + } + + client = s3.NewFromConfig(cfg, func(options *s3.Options) { + if endpoint != "" { + options.BaseEndpoint = aws.String(endpoint) + } + }) + } else { + client = s3.New(s3.Options{ + Credentials: credentials.NewStaticCredentialsProvider(ak, sk, ""), + UsePathStyle: usePathStyle, + Region: region, + EndpointResolver: s3.EndpointResolverFunc( + func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: endpoint, + HostnameImmutable: false, + SigningName: "s3", + PartitionID: "aws", + SigningRegion: region, + SigningMethod: "v4", + Source: aws.EndpointSourceCustom, + }, nil + }), + }) + } + + // check bucket + _, err = client.HeadBucket(context.TODO(), &s3.HeadBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NotFound" { + _, err = client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + return nil, oss.ErrProviderInit.WithError(err) + } + } + } + return &S3Storage{bucket: bucket, client: client}, nil +} + +func (s *S3Storage) Save(key string, data []byte) error { + _, err := s.client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + Body: bytes.NewReader(data), + }) + return err +} + +func (s *S3Storage) Load(key string) ([]byte, error) { + resp, err := s.client.GetObject(context.TODO(), &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, err + } + + return io.ReadAll(resp.Body) +} + +func (s *S3Storage) Exists(key string) (bool, error) { + _, err := s.client.HeadObject(context.TODO(), &s3.HeadObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + return err == nil, nil +} + +func (s *S3Storage) Delete(key string) error { + _, err := s.client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + return err +} + +func (s *S3Storage) List(prefix string) ([]oss.OSSPath, error) { + // append a slash to the prefix if it doesn't end with one + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + + var keys []oss.OSSPath + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(s.bucket), + Prefix: aws.String(prefix), + } + + paginator := s3.NewListObjectsV2Paginator(s.client, input) + for paginator.HasMorePages() { + page, err := paginator.NextPage(context.TODO()) + if err != nil { + return nil, err + } + for _, obj := range page.Contents { + // remove prefix + key := strings.TrimPrefix(*obj.Key, prefix) + // remove leading slash + key = strings.TrimPrefix(key, "/") + keys = append(keys, oss.OSSPath{ + Path: key, + IsDir: false, + }) + } + } + + return keys, nil +} + +func (s *S3Storage) State(key string) (oss.OSSState, error) { + resp, err := s.client.HeadObject(context.TODO(), &s3.HeadObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + if err != nil { + return oss.OSSState{}, err + } + + if resp.ContentLength == nil { + resp.ContentLength = ToPtr[int64](0) + } + if resp.LastModified == nil { + resp.LastModified = ToPtr(time.Time{}) + } + + return oss.OSSState{ + Size: *resp.ContentLength, + LastModified: *resp.LastModified, + }, nil +} + +func (s *S3Storage) Type() string { + return oss.OSS_TYPE_S3 +} + +func ToPtr[T any](value T) *T { + return &value +} diff --git a/oss/tencentcos/cos.go b/oss/tencentcos/cos.go new file mode 100644 index 0000000..b6ae5e1 --- /dev/null +++ b/oss/tencentcos/cos.go @@ -0,0 +1,178 @@ +package tencentcos + +import ( + "bytes" + "context" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/langgenius/dify-cloud-kit/oss" + "github.com/tencentyun/cos-go-sdk-v5" +) + +type TencentCOSStorage struct { + bucket string + region string + client *cos.Client +} + +func NewTencentCOSStorage(args oss.OSSArgs) (oss.OSS, error) { + if args.TencentCOS == nil { + return nil, oss.ErrArgumentInvalid.WithDetail("can't find Tencent COS argument in OSSArgs") + } + + err := args.TencentCOS.Validate() + if err != nil { + return nil, err + } + + bucket := args.TencentCOS.Bucket + region := args.TencentCOS.Region + secretID := args.TencentCOS.SecretID + secretKey := args.TencentCOS.SecretKey + u, err := url.Parse("https://" + bucket + ".cos." + region + ".myqcloud.com") + if err != nil { + return nil, oss.ErrProviderInit.WithError(err).WithDetail("url parse failed") + } + + b := &cos.BaseURL{BucketURL: u} + client := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: secretID, + SecretKey: secretKey, + }, + }) + + _, err = client.Bucket.Head(context.Background()) + if err != nil { + return nil, oss.ErrProviderInit.WithError(err) + } + + return &TencentCOSStorage{ + bucket: bucket, + region: region, + client: client, + }, nil +} + +func (s *TencentCOSStorage) Save(key string, data []byte) error { + _, err := s.client.Object.Put(context.Background(), key, bytes.NewReader(data), nil) + return err +} + +func (s *TencentCOSStorage) Load(key string) ([]byte, error) { + resp, err := s.client.Object.Get(context.Background(), key, nil) + if err != nil { + return nil, err + } + + return io.ReadAll(resp.Body) +} + +func (s *TencentCOSStorage) Exists(key string) (bool, error) { + ok, err := s.client.Object.IsExist(context.Background(), key) + if err == nil && ok { + return true, nil + } else if err != nil { + return false, err + } else { + return false, nil + } +} + +func (s *TencentCOSStorage) Delete(key string) error { + _, err := s.client.Object.Delete(context.Background(), key) + return err +} + +func (s *TencentCOSStorage) List(prefix string) ([]oss.OSSPath, error) { + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + + var keys []oss.OSSPath + opt := &cos.BucketGetOptions{ + Prefix: prefix, + Delimiter: "/", + } + isTruncated := true + var marker string + for isTruncated { + if marker != "" { + opt.Marker = marker + } + + result, _, err := s.client.Bucket.Get(context.Background(), opt) + if err != nil { + return nil, err + } + + for _, content := range result.Contents { + // remove prefix + key := strings.TrimPrefix(content.Key, prefix) + // remove leading slash + key = strings.TrimPrefix(key, "/") + if key == "" { + continue + } + keys = append(keys, oss.OSSPath{ + Path: key, + IsDir: false, + }) + } + + for _, commonPrefix := range result.CommonPrefixes { + if commonPrefix == "" { + continue + } + if !strings.HasSuffix(commonPrefix, "/") { + commonPrefix = commonPrefix + "/" + } + keys = append(keys, oss.OSSPath{ + Path: commonPrefix, + IsDir: true, + }) + + subKeys, _ := s.List(commonPrefix) + if len(subKeys) > 0 { + subPrefix := strings.TrimPrefix(commonPrefix, prefix) + for i := range subKeys { + subKeys[i].Path = subPrefix + subKeys[i].Path + } + keys = append(keys, subKeys...) + } + + } + + isTruncated = result.IsTruncated + marker = result.NextMarker + } + + return keys, nil +} + +func (s *TencentCOSStorage) State(key string) (oss.OSSState, error) { + resp, err := s.client.Object.Head(context.Background(), key, nil) + if err != nil { + return oss.OSSState{}, err + } + + contentLength := resp.ContentLength + + lastModified, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified")) + if err != nil { + lastModified = time.Time{} + } + + return oss.OSSState{ + Size: contentLength, + LastModified: lastModified, + }, nil +} + +func (s *TencentCOSStorage) Type() string { + return oss.OSS_TYPE_TENCENT_COS +} diff --git a/tests/oss/oss_test.go b/tests/oss/oss_test.go new file mode 100644 index 0000000..becf3c2 --- /dev/null +++ b/tests/oss/oss_test.go @@ -0,0 +1,173 @@ +package main + +import ( + "fmt" + "log" + "math/rand" + "os" + "testing" + "time" + + "github.com/langgenius/dify-cloud-kit/oss" + "github.com/langgenius/dify-cloud-kit/oss/factory" + "github.com/stretchr/testify/assert" +) + +type testArgsCases struct { + vendor string + args oss.OSSArgs + skip bool +} + +var allCases = []testArgsCases{ + { + vendor: "local", + args: oss.OSSArgs{ + Local: &oss.Local{ + Path: "/tmp/dify-oss-tests", + }, + }, + skip: false, + }, + { + vendor: "s3", + args: oss.OSSArgs{ + S3: &oss.S3{ + UseAws: true, + UsePathStyle: true, + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Bucket: os.Getenv("AWS_S3_BUCKET"), + Region: os.Getenv("AWS_S3_REGION"), + }, + }, + skip: false, + }, + { + vendor: "azure", + args: oss.OSSArgs{ + AzureBlob: &oss.AzureBlob{ + ConnectionString: os.Getenv("AZURE_CONNECTION"), + ContainerName: os.Getenv("AZURE_CONTAINER"), + }, + }, + skip: false, + }, + { + vendor: "aliyun", + args: oss.OSSArgs{ + AliyunOSS: &oss.AliyunOSS{ + Region: os.Getenv("ALIYUN_OSS_REGION"), + Endpoint: os.Getenv("ALIYUN_OSS_ENDPOINT"), + AccessKey: os.Getenv("ALIYUN_OSS_ACCESS_KEY"), + SecretKey: os.Getenv("ALIYUN_OSS_SECRET_KEY"), + AuthVersion: os.Getenv("ALIYUN_OSS_AUTH_VERSION"), + Path: os.Getenv("ALIYUN_OSS_PATH"), + Bucket: os.Getenv("ALIYUN_OSS_BUCKET"), + }, + }, + skip: false, + }, + { + vendor: "tencent", + args: oss.OSSArgs{ + TencentCOS: &oss.TencentCOS{ + Region: os.Getenv("TENCNET_COS_REGION"), + SecretID: os.Getenv("TENCNET_COS_SECRET_ID"), + SecretKey: os.Getenv("TENCNET_COS_SECRET_KEY"), + Bucket: os.Getenv("TENCNET_COS_BUCKET"), + }, + }, + skip: false, + }, + { + vendor: "gcs", + args: oss.OSSArgs{ + GoogleCloudStorage: &oss.GoogleCloudStorage{ + Bucket: os.Getenv("GCS_BUCKET"), + CredentialsB64: os.Getenv("GCS_CREDENTIALS"), + }, + }, + skip: false, + }, + { + vendor: "huawei", + args: oss.OSSArgs{ + HuaweiOBS: &oss.HuaweiOBS{ + Bucket: os.Getenv("HUAWEI_OBS_BUCKET"), + AccessKey: os.Getenv("HUAWEI_OBS_ACCESS_KEY"), + SecretKey: os.Getenv("HUAWEI_OBS_SECRET_KEY"), + Server: os.Getenv("HUAWEI_OBS_SERVER"), + }, + }, + skip: false, + }, +} + +const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +func randomString(length int) string { + seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) + result := make([]byte, length) + for i := range result { + result[i] = charset[seededRand.Intn(len(charset))] + } + return string(result) +} + +func TestAll(t *testing.T) { + prefix := randomString(5) + key := fmt.Sprintf("%s/%s", prefix, randomString(10)) + + size := 1 * 1024 * 1024 + data := make([]byte, size) + + for _, c := range allCases { + if c.skip { + continue + } + storage, err := factory.Load(c.vendor, c.args) + if err != nil { + log.Fatal(err) + continue + } + info := fmt.Sprintf("vendor: %s", c.vendor) + ossPaths, err := storage.List(prefix) + assert.Equal(t, 0, len(ossPaths), info) + assert.Nil(t, err, info) + + exist, err := storage.Exists(key) + assert.Equal(t, false, exist, info) + assert.Nil(t, err, info) + + err = storage.Save(key, data) + assert.Nil(t, err, info) + + rdata, err := storage.Load(key) + assert.Equal(t, data, rdata, info) + assert.Nil(t, err, info) + + ossState, err := storage.State(key) + assert.Equal(t, int64(size), ossState.Size, info) + assert.Nil(t, err, info) + + exist, err = storage.Exists(key) + assert.Equal(t, true, exist, info) + assert.Nil(t, err, info) + + ossPaths, err = storage.List(prefix) + assert.Equal(t, 1, len(ossPaths), info) + assert.Nil(t, err, info) + + err = storage.Delete(key) + assert.Nil(t, err, info) + + exist, err = storage.Exists(key) + assert.Equal(t, false, exist, info) + assert.Nil(t, err, info) + + ossPaths, err = storage.List(prefix) + assert.Equal(t, 0, len(ossPaths), info) + assert.Nil(t, err, info) + } +}