Skip to content

Commit b3feaff

Browse files
tishin-endoutishin-endou
committed
feat(provider): add s3compatsigv4 provider for S3 signature V4
fix admin test case and issues according to feedback
1 parent 443c490 commit b3feaff

61 files changed

Lines changed: 3365 additions & 5 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ COPY ./addons/swift/requirements.txt ./addons/swift/
6161
COPY ./addons/azureblobstorage/requirements.txt ./addons/azureblobstorage/
6262
COPY ./addons/weko/requirements.txt ./addons/weko/
6363
COPY ./addons/s3compat/requirements.txt ./addons/s3compat/
64+
COPY ./addons/s3compatsigv4/requirements.txt ./addons/s3compatsigv4/
6465
COPY ./addons/s3compatinstitutions/requirements.txt ./addons/s3compatinstitutions/
6566
COPY ./addons/s3compatb3/requirements.txt ./addons/s3compatb3/
6667
COPY ./addons/ociinstitutions/requirements.txt ./addons/ociinstitutions/
@@ -167,6 +168,7 @@ COPY ./addons/azureblobstorage/static/ ./addons/azureblobstorage/static/
167168
COPY ./addons/weko/static/ ./addons/weko/static/
168169
COPY ./addons/jupyterhub/static/ ./addons/jupyterhub/static/
169170
COPY ./addons/s3compat/static/ ./addons/s3compat/static/
171+
COPY ./addons/s3compatsigv4/static/ ./addons/s3compatsigv4/static/
170172
COPY ./addons/s3compatinstitutions/static/ ./addons/s3compatinstitutions/static/
171173
COPY ./addons/s3compatb3/static/ ./addons/s3compatb3/static/
172174
COPY ./addons/ociinstitutions/requirements.txt ./addons/ociinstitutions/

‎addons.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
"weko",
2323
"jupyterhub",
2424
"s3compat",
25+
"s3compatsigv4",
2526
"s3compatinstitutions",
2627
"s3compatb3",
2728
"ociinstitutions",
@@ -61,6 +62,7 @@
6162
"azureblobstorage": "partial",
6263
"weko": "partial",
6364
"s3compat": "partial",
65+
"s3compatsigv4": "partial",
6466
"s3compatinstitutions": "partial",
6567
"s3compatb3": "partial",
6668
"ociinstitutions": "partial",
@@ -88,6 +90,7 @@
8890
"azureblobstorage",
8991
"weko",
9092
"s3compat",
93+
"s3compatsigv4",
9194
"s3compatinstitutions",
9295
"s3compatb3",
9396
"ociinstitutions",
@@ -128,6 +131,7 @@
128131
"weko": "JAIRO Cloud is an application server to share, archive data.",
129132
"jupyterhub": "Jupyter is a web-based interactive computational environment. Files on a GakuNin RDM project can be imported to/exported from Jupyter",
130133
"s3compat": "S3 Compatible Storage is a file storage add-on. Connect your S3 Compatible Storage account to a GakuNin RDM project to interact with files hosted on S3 Compatible Storage via the GakuNin RDM.",
134+
"s3compatsigv4": "S3 Compatible Storage (SigV4) is a file storage add-on. Connect your S3 Compatible Storage (SigV4) account to a GakuNin RDM project to interact with files hosted on S3 Compatible Storage (SigV4) via the GakuNin RDM.",
131135
"s3compatinstitutions": "S3 Compatible Storage for Institutions is a file storage add-on. Connect your S3 Compatible Storage account to a GakuNin RDM project to interact with files hosted on S3 Compatible Storage via the GakuNin RDM.",
132136
"s3compatb3": "S3 Compatible Storage is a file storage add-on. Connect your S3 Compatible Storage account to a GakuNin RDM project to interact with files hosted on S3 Compatible Storage via the GakuNin RDM.",
133137
"ociinstitutions": "Oracle Cloud Infrastructure for Institutions is a file storage add-on. Connect your Oracle Cloud Infrastructure Object Storage account to a GakuNin RDM project to interact with files hosted on Oracle Cloud Infrastructure Object Storage via the GakuNin RDM.",
@@ -159,6 +163,7 @@
159163
"weko": "https://weko.at.nii.ac.jp/",
160164
"jupyterhub": "https://jupyterhub.readthedocs.io/",
161165
"s3compat": "https://aws.amazon.com/s3/",
166+
"s3compatsigv4": "https://aws.amazon.com/s3/",
162167
"s3compatb3": "https://aws.amazon.com/s3/",
163168
"nextcloud": "https://nextcloud.com/",
164169
"iqbrims": "https://drive.google.com",
@@ -184,18 +189,21 @@
184189
"onedrive",
185190
"s3",
186191
"s3compat",
192+
"s3compatsigv4",
187193
"owncloud"
188194
],
189195
"addons_has_max_keys": [
190196
"s3",
191197
"s3compat",
198+
"s3compatsigv4",
192199
"s3compatinstitutions",
193200
"s3compatb3",
194201
"ociinstitutions"
195202
],
196203
"addons_folder_field": {
197204
"s3": "folder_name",
198205
"s3compat": "folder_name",
206+
"s3compatsigv4": "folder_name",
199207
"s3compatb3": "folder_name",
200208
"azureblobstorage": "folder_name",
201209
"box": "folder_name",

‎addons/s3compatsigv4/README.md‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# RDM S3 Compatible Storage (SigV4) Addon
2+
3+
S3 Compatible Storage (SigV4) Addon enables to mount Cloud Storage which supports Amazon S3-like API on the project.
4+
5+
## Configuring the addon
6+
7+
Users can select storage from the S3 Compatible Storage (SigV4) List,
8+
which is defined in `addons/s3compatsigv4/static/settings.json`.
9+
10+
```
11+
{
12+
"availableServices": [{"name": "Wasabi",
13+
"host": "s3.wasabisys.com",
14+
"bucketLocations": {
15+
"us-east": {"name": "us-east", "host": "s3.wasabisys.com"},
16+
"us-west-1": {"name": "us-west-1", "host": "s3.us-west-1.wasabisys.com"},
17+
"eu-central": {"name": "eu-central", "host": "s3.eu-central-1.wasabisys.com"},
18+
"": {"name": "Virginia"}}},
19+
{"name": "My Private Storage",
20+
"host": "my-private-storage-address:80"}
21+
],
22+
"encryptUploads": true
23+
}
24+
```
25+
26+
## Enabling the addon
27+
28+
### Enable on RDM
29+
1. On RDM, enable S3 Compatible Storage (SigV4) as a provider
30+
2. Scroll down to Configure Add-ons
31+
3. Choose desired storage service
32+
4. Connect your account and enter your ID and secret
33+
5. Select a bucket to work from, or create a new one.

‎addons/s3compatsigv4/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
default_app_config = 'addons.s3compatsigv4.apps.S3CompatSigV4AddonAppConfig'

‎addons/s3compatsigv4/apps.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import os
2+
from addons.base.apps import BaseAddonAppConfig, generic_root_folder
3+
from addons.s3compatsigv4.settings import MAX_UPLOAD_SIZE
4+
5+
s3compatsigv4_root_folder = generic_root_folder('s3compatsigv4')
6+
7+
HERE = os.path.dirname(os.path.abspath(__file__))
8+
TEMPLATE_PATH = os.path.join(
9+
HERE,
10+
'templates'
11+
)
12+
13+
class S3CompatSigV4AddonAppConfig(BaseAddonAppConfig):
14+
15+
name = 'addons.s3compatsigv4'
16+
label = 'addons_s3compatsigv4'
17+
owners = ['user', 'node']
18+
configs = ['accounts', 'node']
19+
categories = ['storage']
20+
has_hgrid_files = True
21+
max_file_size = MAX_UPLOAD_SIZE
22+
node_settings_template = os.path.join(TEMPLATE_PATH, 's3compatsigv4_node_settings.mako')
23+
user_settings_template = os.path.join(TEMPLATE_PATH, 's3compatsigv4_user_settings.mako')
24+
25+
@property
26+
def full_name(self):
27+
return 'S3 Compatible Storage (SigV4)'
28+
29+
@property
30+
def short_name(self):
31+
return 's3compatsigv4'
32+
33+
@property
34+
def get_hgrid_data(self):
35+
return s3compatsigv4_root_folder
36+
37+
BUCKET_LINKED = 's3compatsigv4_bucket_linked'
38+
BUCKET_UNLINKED = 's3compatsigv4_bucket_unlinked'
39+
FILE_ADDED = 's3compatsigv4_file_added'
40+
FILE_REMOVED = 's3compatsigv4_file_removed'
41+
FILE_UPDATED = 's3compatsigv4_file_updated'
42+
FOLDER_CREATED = 's3compatsigv4_folder_created'
43+
NODE_AUTHORIZED = 's3compatsigv4_node_authorized'
44+
NODE_DEAUTHORIZED = 's3compatsigv4_node_deauthorized'
45+
NODE_DEAUTHORIZED_NO_USER = 's3compatsigv4_node_deauthorized_no_user'
46+
47+
actions = (BUCKET_LINKED,
48+
BUCKET_UNLINKED,
49+
FILE_ADDED,
50+
FILE_REMOVED,
51+
FILE_UPDATED,
52+
FOLDER_CREATED,
53+
NODE_AUTHORIZED,
54+
NODE_DEAUTHORIZED,
55+
NODE_DEAUTHORIZED_NO_USER)
56+
57+
@property
58+
def routes(self):
59+
from . import routes
60+
return [routes.api_routes]
61+
62+
@property
63+
def user_settings(self):
64+
return self.get_model('UserSettings')
65+
66+
@property
67+
def node_settings(self):
68+
return self.get_model('NodeSettings')

‎addons/s3compatsigv4/migrations/0001_initial.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# -*- coding: utf-8 -*-
2+
# Generated by Django 1.11.28 on 2026-01-12 14:02
3+
from __future__ import unicode_literals
4+
5+
import addons.base.models
6+
from django.conf import settings
7+
from django.db import migrations, models
8+
import django.db.models.deletion
9+
import django_extensions.db.fields
10+
import osf.models.base
11+
import osf.utils.datetime_aware_jsonfield
12+
import osf.utils.fields
13+
14+
15+
class Migration(migrations.Migration):
16+
17+
initial = True
18+
19+
dependencies = [
20+
('osf', '0261_auto_20260112_1402'),
21+
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
22+
]
23+
24+
operations = [
25+
migrations.CreateModel(
26+
name='NodeSettings',
27+
fields=[
28+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
29+
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
30+
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
31+
('_id', models.CharField(db_index=True, default=osf.models.base.generate_object_id, max_length=24, unique=True)),
32+
('is_deleted', models.BooleanField(default=False)),
33+
('deleted', osf.utils.fields.NonNaiveDateTimeField(blank=True, null=True)),
34+
('folder_id', models.TextField(blank=True, null=True)),
35+
('folder_name', models.TextField(blank=True, null=True)),
36+
('folder_location', models.TextField(blank=True, null=True)),
37+
('encrypt_uploads', models.BooleanField(default=True)),
38+
('external_account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='addons_s3compatsigv4_node_settings', to='osf.ExternalAccount')),
39+
('owner', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='addons_s3compatsigv4_node_settings', to='osf.AbstractNode')),
40+
],
41+
options={
42+
'abstract': False,
43+
},
44+
bases=(models.Model, osf.models.base.QuerySetExplainMixin, addons.base.models.BaseStorageAddon),
45+
),
46+
migrations.CreateModel(
47+
name='UserSettings',
48+
fields=[
49+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
50+
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
51+
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
52+
('_id', models.CharField(db_index=True, default=osf.models.base.generate_object_id, max_length=24, unique=True)),
53+
('is_deleted', models.BooleanField(default=False)),
54+
('deleted', osf.utils.fields.NonNaiveDateTimeField(blank=True, null=True)),
55+
('oauth_grants', osf.utils.datetime_aware_jsonfield.DateTimeAwareJSONField(blank=True, default=dict, encoder=osf.utils.datetime_aware_jsonfield.DateTimeAwareJSONEncoder)),
56+
('owner', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='addons_s3compatsigv4_user_settings', to=settings.AUTH_USER_MODEL)),
57+
],
58+
options={
59+
'abstract': False,
60+
},
61+
bases=(models.Model, osf.models.base.QuerySetExplainMixin),
62+
),
63+
migrations.AddField(
64+
model_name='nodesettings',
65+
name='user_settings',
66+
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='addons_s3compatsigv4.UserSettings'),
67+
),
68+
]

‎addons/s3compatsigv4/migrations/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)