Skip to content

Commit caa2b3a

Browse files
committed
fix: add cloud OSS storage provider
1 parent f07094c commit caa2b3a

5 files changed

Lines changed: 149 additions & 5 deletions

File tree

object/provider.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ func (provider *Provider) GetId() string {
310310
}
311311

312312
func (p *Provider) GetStorageProviderObj(vectorStoreId string, lang string) (storage.StorageProvider, error) {
313-
pProvider, err := storage.GetStorageProvider(p.Type, p.ClientId, p.ClientSecret, p.Name, vectorStoreId, lang)
313+
pProvider, err := storage.GetStorageProvider(p.Type, p.ClientId, p.ClientSecret, p.Region, p.Domain, p.ProviderUrl, p.Name, vectorStoreId, lang)
314314
if err != nil {
315315
return nil, err
316316
}

storage/aliyun_oss.go

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
// Copyright 2023 The OpenAgent Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package storage
16+
17+
import (
18+
"bytes"
19+
"context"
20+
"fmt"
21+
"strings"
22+
"time"
23+
24+
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
25+
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
26+
)
27+
28+
type AliyunOssStorageProvider struct {
29+
client *oss.Client
30+
bucket string
31+
endpoint string
32+
}
33+
34+
func NewAliyunOssStorageProvider(accessKeyId string, accessKeySecret string, region string, bucket string, endpoint string) (*AliyunOssStorageProvider, error) {
35+
cfg := oss.LoadDefaultConfig().
36+
WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyId, accessKeySecret, "")).
37+
WithRegion(region).
38+
WithEndpoint(endpoint)
39+
40+
client := oss.NewClient(cfg)
41+
return &AliyunOssStorageProvider{
42+
client: client,
43+
bucket: bucket,
44+
endpoint: endpoint,
45+
}, nil
46+
}
47+
48+
func (p *AliyunOssStorageProvider) getObjectUrl(key string) string {
49+
endpoint := p.endpoint
50+
if !strings.HasPrefix(endpoint, "http") {
51+
endpoint = "https://" + endpoint
52+
}
53+
endpoint = strings.TrimRight(endpoint, "/")
54+
return fmt.Sprintf("%s/%s/%s", endpoint, p.bucket, key)
55+
}
56+
57+
func (p *AliyunOssStorageProvider) ListObjects(prefix string) ([]*Object, error) {
58+
result, err := p.client.ListObjectsV2(context.TODO(), &oss.ListObjectsV2Request{
59+
Bucket: oss.Ptr(p.bucket),
60+
Prefix: oss.Ptr(prefix),
61+
})
62+
if err != nil {
63+
return nil, err
64+
}
65+
66+
var objects []*Object
67+
for _, obj := range result.Contents {
68+
key := oss.ToString(obj.Key)
69+
size := obj.Size
70+
lastModified := ""
71+
if obj.LastModified != nil {
72+
lastModified = obj.LastModified.Format(time.RFC3339)
73+
}
74+
objects = append(objects, &Object{
75+
Key: key,
76+
LastModified: lastModified,
77+
Size: size,
78+
Url: p.getObjectUrl(key),
79+
})
80+
}
81+
return objects, nil
82+
}
83+
84+
func (p *AliyunOssStorageProvider) PutObject(user string, parent string, key string, fileBuffer *bytes.Buffer) (string, error) {
85+
_, err := p.client.PutObject(context.TODO(), &oss.PutObjectRequest{
86+
Bucket: oss.Ptr(p.bucket),
87+
Key: oss.Ptr(key),
88+
Body: fileBuffer,
89+
})
90+
if err != nil {
91+
return "", err
92+
}
93+
return p.getObjectUrl(key), nil
94+
}
95+
96+
func (p *AliyunOssStorageProvider) DeleteObject(key string) error {
97+
_, err := p.client.DeleteObject(context.TODO(), &oss.DeleteObjectRequest{
98+
Bucket: oss.Ptr(p.bucket),
99+
Key: oss.Ptr(key),
100+
})
101+
return err
102+
}

storage/provider.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,13 @@ type StorageProvider interface {
2929
DeleteObject(key string) error
3030
}
3131

32-
func GetStorageProvider(typ string, clientId string, clientSecret string, providerName string, vectorStoreId string, lang string) (StorageProvider, error) {
32+
func GetStorageProvider(typ string, clientId string, clientSecret string, region string, bucket string, endpoint string, providerName string, vectorStoreId string, lang string) (StorageProvider, error) {
3333
var p StorageProvider
3434
var err error
3535
if typ == "Local File System" {
3636
p, err = NewLocalFileSystemStorageProvider(clientId)
37+
} else if typ == "Alibaba Cloud OSS" {
38+
p, err = NewAliyunOssStorageProvider(clientId, clientSecret, region, bucket, endpoint)
3739
} else {
3840
p, err = NewCasdoorProvider(providerName, lang)
3941
}

web/src/ProviderEditPage.js

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@ class ProviderEditPage extends React.Component {
7272
}
7373
}
7474
if (provider.category === "Storage") {
75+
if (provider.type === "Alibaba Cloud OSS") {
76+
return Setting.getLabel(i18next.t("provider:Client ID"), i18next.t("provider:Client ID - Tooltip"));
77+
}
7578
return Setting.getLabel(i18next.t("store:Storage subpath"), i18next.t("store:Storage subpath - Tooltip"));
7679
}
7780
return Setting.getLabel(i18next.t("provider:Client ID"), i18next.t("provider:Client ID - Tooltip"));
@@ -145,7 +148,7 @@ class ProviderEditPage extends React.Component {
145148

146149
shouldShowClientSecretInput(provider) {
147150
return !(
148-
provider.category === "Storage" ||
151+
(provider.category === "Storage" && provider.type !== "Alibaba Cloud OSS") ||
149152
(provider.category === "Blockchain" && provider.type === "ChainMaker") ||
150153
provider.type === "Dummy" ||
151154
provider.type === "Ollama"
@@ -709,6 +712,42 @@ class ProviderEditPage extends React.Component {
709712
</>
710713
) : null
711714
}
715+
{
716+
(this.state.provider.category === "Storage" && this.state.provider.type === "Alibaba Cloud OSS") ? (
717+
<>
718+
<Row style={{marginTop: "20px"}} >
719+
<Col style={{marginTop: "5px"}} span={(Setting.isMobile()) ? 22 : 2}>
720+
{Setting.getLabel(i18next.t("general:Region"), i18next.t("general:Region - Tooltip"))}
721+
</Col>
722+
<Col span={22} >
723+
<Input disabled={isRemote} value={this.state.provider.region} onChange={e => {
724+
this.updateProviderField("region", e.target.value);
725+
}} />
726+
</Col>
727+
</Row>
728+
<Row style={{marginTop: "20px"}} >
729+
<Col style={{marginTop: "5px"}} span={(Setting.isMobile()) ? 22 : 2}>
730+
{Setting.getLabel(i18next.t("provider:Bucket"), i18next.t("provider:Bucket - Tooltip"))}
731+
</Col>
732+
<Col span={22} >
733+
<Input disabled={isRemote} value={this.state.provider.domain} onChange={e => {
734+
this.updateProviderField("domain", e.target.value);
735+
}} />
736+
</Col>
737+
</Row>
738+
<Row style={{marginTop: "20px"}} >
739+
<Col style={{marginTop: "5px"}} span={(Setting.isMobile()) ? 22 : 2}>
740+
{Setting.getLabel(i18next.t("general:Endpoint"), i18next.t("general:Endpoint - Tooltip"))}
741+
</Col>
742+
<Col span={22} >
743+
<Input disabled={isRemote} value={this.state.provider.providerUrl} onChange={e => {
744+
this.updateProviderField("providerUrl", e.target.value);
745+
}} />
746+
</Col>
747+
</Row>
748+
</>
749+
) : null
750+
}
712751
{
713752
["Storage", "Model", "Embedding", "Text-to-Speech", "Speech-to-Text"].includes(this.state.provider.category) || (this.state.provider.category === "Blockchain" && this.state.provider.type === "Ethereum") ? null : (
714753
<Row style={{marginTop: "20px"}} >
@@ -1143,7 +1182,7 @@ class ProviderEditPage extends React.Component {
11431182
/>
11441183
</Card>
11451184
{
1146-
(this.state.provider.category !== "Model" || this.modelCategoryShowsProviderUrlInput(this.state.provider.type)) ? (
1185+
(this.state.provider.category !== "Model" || this.modelCategoryShowsProviderUrlInput(this.state.provider.type)) && !(this.state.provider.category === "Storage" && this.state.provider.type === "Alibaba Cloud OSS") ? (
11471186
<Row style={{marginTop: "20px"}} >
11481187
<Col style={{marginTop: "5px"}} span={(Setting.isMobile()) ? 22 : 2}>
11491188
{this.getProviderUrlLabel(this.state.provider)}

web/src/Setting.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1023,7 +1023,7 @@ export function getOtherProviderInfo() {
10231023
logo: "https://min.io/resources/img/logo.svg",
10241024
url: "https://min.io/",
10251025
},
1026-
"Aliyun OSS": {
1026+
"Alibaba Cloud OSS": {
10271027
logo: `${StaticBaseUrl}/img/social_aliyun.png`,
10281028
url: "https://aliyun.com/product/oss",
10291029
},
@@ -1301,6 +1301,7 @@ export function getProviderTypeOptions(category) {
13011301
return (
13021302
[
13031303
{id: "Local File System", name: "Local File System"},
1304+
{id: "Alibaba Cloud OSS", name: "Alibaba Cloud OSS"},
13041305
]
13051306
);
13061307
} else if (category === "Model") {

0 commit comments

Comments
 (0)