Merge pull request #513 from singnet/development

Added FileCoin storage provider for services metadata and files

Author: sassless

requirements.txt

@@ -20,3 +20,4 @@ eth-account==0.9.0
 trezor==0.13.8
 ledgerblue==0.1.48
 snet.contracts==0.1.1
+lighthouseweb3==0.1.4

snet/cli/arguments.py

@@ -2,6 +2,8 @@
 import os
 import re
 import sys
+from email.policy import default
+from random import choices
 
 from snet.contracts import get_all_abi_contract_files, get_contract_def
 
@@ -258,6 +260,15 @@ def add_metadatafile_argument_for_org(p):
                    help="Service metadata json file (default organization_metadata.json)")
 
 
+def add_p_storage_param(_p):
+    _p.add_argument(
+        "--storage",
+        default="ipfs",
+        choices=['ipfs', 'filecoin'],
+        help="Choose storage for uploading metadata/protobuf file (defaults to 'ipfs')",
+    )
+
+
 def add_organization_options(parser):
     parser.set_defaults(cmd=OrganizationCommand)
 
@@ -390,8 +401,9 @@ def add_organization_options(parser):
     add_metadatafile_argument_for_org(p)
     p.add_argument("--members", help="List of members to be added to the organization", metavar="ORG_MEMBERS[]")
     add_organization_arguments(p)
+    add_p_storage_param(p)
 
-    p = subparsers.add_parser("update-metadata", help="Create an Organization")
+    p = subparsers.add_parser("update-metadata", help="Update metadata for an Organization")
     p.add_argument("org_id", help="Unique Organization Id", metavar="ORG_ID")
     p.set_defaults(fn="update_metadata")
     add_metadatafile_argument_for_org(p)
@@ -400,6 +412,7 @@ def add_organization_options(parser):
         help="List of members to be added to the organization",
         metavar="ORG_MEMBERS[]")
     add_organization_arguments(p)
+    add_p_storage_param(p)
 
     p = subparsers.add_parser("change-owner", help="Change Organization's owner")
     p.set_defaults(fn="change_owner")
@@ -936,7 +949,7 @@ def add_mpe_service_options(parser):
     add_p_metadata_file_opt(p)
 
     p = subparsers.add_parser("metadata-init",
-                              help="Init metadata file with providing protobuf directory (which we publish in IPFS) and display_name (optionally encoding, service_type and payment_expiration_threshold)")
+                              help="Init metadata file with providing protobuf directory (which we publish in IPFS or FileCoin) and display_name (optionally encoding, service_type and payment_expiration_threshold)")
     p.set_defaults(fn="publish_proto_metadata_init")
     p.add_argument("protodir",
                    help="Directory which contains protobuf files",
@@ -965,14 +978,16 @@ def add_mpe_service_options(parser):
                    default="grpc",
                    choices=['grpc', 'http', 'jsonrpc', 'process'],
                    help="Service type")
+    add_p_storage_param(p)
 
-    p = subparsers.add_parser("metadata-set-model",
-                              help="Publish protobuf model in ipfs and update existed metadata file")
+    p = subparsers.add_parser("metadata-set-api",
+                              help="Publish protobuf model in ipfs or filecoin and update existed metadata file")
     p.set_defaults(fn="publish_proto_metadata_update")
     p.add_argument("protodir",
                    help="Directory which contains protobuf files",
                    metavar="PROTO_DIR")
     add_p_metadata_file_opt(p)
+    add_p_storage_param(p)
 
     p = subparsers.add_parser("metadata-set-fixed-price",
                               help="Set pricing model as fixed price for all methods")
@@ -1199,6 +1214,7 @@ def add_p_publish_params(_p):
         _p.add_argument("--update-mpe-address",
                         action='store_true',
                         help="Update mpe_address in metadata before publishing them")
+        add_p_storage_param(p)
         add_p_mpe_address_opt(_p)
 
     p = subparsers.add_parser("publish",
@@ -1214,9 +1230,15 @@ def add_p_publish_params(_p):
     add_p_publish_params(p)
     add_transaction_arguments(p)
 
+    p = subparsers.add_parser("publish-in-filecoin",
+                              help="Publish metadata only in FileCoin, without publishing in Registry")
+    p.set_defaults(fn="publish_metadata_in_filecoin")
+    add_p_publish_params(p)
+    add_transaction_arguments(p)
+
     p = subparsers.add_parser("update-metadata",
-                              help="Publish metadata in IPFS and update existed service")
-    p.set_defaults(fn="publish_metadata_in_ipfs_and_update_registration")
+                              help="Publish metadata in IPFS or FileCoin and update existed service")
+    p.set_defaults(fn="publish_metadata_in_storage_and_update_registration")
     add_p_publish_params(p)
     add_p_service_in_registry(p)
     add_transaction_arguments(p)
@@ -1286,7 +1308,7 @@ def add_p_protodir_to_extract(_p):
                         metavar="PROTO_DIR")
 
     p = subparsers.add_parser("get-api-metadata",
-                              help="Extract service api (model) to the given protodir. Get model_ipfs_hash from metadata")
+                              help="Extract service api (model) to the given protodir. Get existed metadata")
     p.set_defaults(fn="extract_service_api_from_metadata")
     add_p_protodir_to_extract(p)
     add_p_metadata_file_opt(p)

snet/cli/commands/commands.py

@@ -7,6 +7,7 @@
 from urllib.parse import urljoin
 
 import ipfshttpclient
+from lighthouseweb3 import Lighthouse
 import yaml
 from rfc3986 import urlparse
 import web3
@@ -18,10 +19,10 @@
 from snet.cli.metadata.organization import OrganizationMetadata, PaymentStorageClient, Payment, Group
 from snet.cli.utils.config import get_contract_address, get_field_from_args_or_session, \
     read_default_contract_address
-from snet.cli.utils.ipfs_utils import bytesuri_to_hash, get_from_ipfs_and_checkhash, \
-    hash_to_bytesuri, publish_file_in_ipfs
+from snet.cli.utils.ipfs_utils import get_from_ipfs_and_checkhash, \
+    hash_to_bytesuri, publish_file_in_ipfs, publish_file_in_filecoin
 from snet.cli.utils.utils import DefaultAttributeObject, get_web3, is_valid_url, serializable, type_converter, \
-    get_cli_version, bytes32_to_str
+    get_cli_version, bytes32_to_str, bytesuri_to_hash, get_file_from_filecoin
 
 
 class Command(object):
@@ -79,6 +80,10 @@ def _get_ipfs_client(self):
         ipfs_endpoint = self.config.get_ipfs_endpoint()
         return ipfshttpclient.connect(ipfs_endpoint)
 
+    def _get_filecoin_client(self):
+        lighthouse_token = self.config.get_filecoin_key()
+        return Lighthouse(token=lighthouse_token)
+
 
 class VersionCommand(Command):
     def show(self):
@@ -489,9 +494,11 @@ def _get_organization_registration(self, org_id):
 
     def _get_organization_metadata_from_registry(self, org_id):
         rez = self._get_organization_registration(org_id)
-        metadata_hash = bytesuri_to_hash(rez["orgMetadataURI"])
-        metadata = get_from_ipfs_and_checkhash(
-            self._get_ipfs_client(), metadata_hash)
+        storage_type, metadata_hash = bytesuri_to_hash(rez["orgMetadataURI"])
+        if storage_type == "ipfs":
+            metadata = get_from_ipfs_and_checkhash(self._get_ipfs_client(), metadata_hash)
+        else:
+            metadata = get_file_from_filecoin(metadata_hash)
         metadata = metadata.decode("utf-8")
         return OrganizationMetadata.from_json(json.loads(metadata))
 
@@ -589,10 +596,17 @@ def create(self):
 
         members = self.get_members_from_args()
 
-        ipfs_metadata_uri = publish_file_in_ipfs(
-            self._get_ipfs_client(), metadata_file, False)
-        params = [type_converter("bytes32")(
-            org_id), hash_to_bytesuri(ipfs_metadata_uri), members]
+        storage = self.args.storage
+        if not storage or storage == "ipfs":
+            metadata_uri = publish_file_in_ipfs(self._get_ipfs_client(), metadata_file, False)
+        elif storage == "filecoin":
+            # upload to Filecoin via Lighthouse SDK
+            metadata_uri = publish_file_in_filecoin(self._get_filecoin_client(), metadata_file)
+        else:
+            raise ValueError(f"Unsupported storage option: {storage}. Use --storage <ipfs or filecoin>")
+
+        params = [type_converter("bytes32")(org_id),
+                  hash_to_bytesuri(metadata_uri, storage), members]
         self._printout("Creating transaction to create organization name={} id={}\n".format(
             org_metadata.org_name, org_id))
         self.transact_contract_command(
@@ -622,31 +636,34 @@ def update_metadata(self):
             with open(metadata_file, 'r') as f:
                 org_metadata = OrganizationMetadata.from_json(json.load(f))
         except Exception as e:
-            print(
-                "Organization metadata json file not found ,Please check --metadata-file path ")
+            print("Organization metadata JSON file not found. Please check --metadata-file path.")
             raise e
-        # validate the metadata before updating
 
+        # Validate the metadata before updating
         org_id = self.args.org_id
-        existing_registry_org_metadata = self._get_organization_metadata_from_registry(
-            org_id)
+        existing_registry_org_metadata = self._get_organization_metadata_from_registry(org_id)
         org_metadata.validate(existing_registry_org_metadata)
 
         # Check if Organization already exists
         found = self._get_organization_by_id(org_id)[0]
         if not found:
-            raise Exception(
-                "\nOrganization with id={} does not  exists!\n".format(org_id))
+            raise Exception("\nOrganization with id={} does not exist!\n".format(org_id))
+
+        storage = self.args.storage
+        if not storage or storage == "ipfs":
+            metadata_uri = publish_file_in_ipfs(self._get_ipfs_client(), metadata_file, False)
+        elif storage == "filecoin":
+            # upload to Filecoin via Lighthouse SDK
+            metadata_uri = publish_file_in_filecoin(self._get_filecoin_client(), metadata_file)
+        else:
+            raise ValueError(f"Unsupported storage option: {storage}. Use --storage <ipfs or filecoin>")
 
-        ipfs_metadata_uri = publish_file_in_ipfs(
-            self._get_ipfs_client(), metadata_file, False)
-        params = [type_converter("bytes32")(
-            org_id), hash_to_bytesuri(ipfs_metadata_uri)]
+        params = [type_converter("bytes32")(org_id), hash_to_bytesuri(metadata_uri, storage)]
         self._printout(
-            "Creating transaction to create organization name={} id={}\n".format(org_metadata.org_name, org_id))
-        self.transact_contract_command(
-            "Registry", "changeOrganizationMetadataURI", params)
-        self._printout("id:\n%s" % org_id)
+            "Creating transaction to update organization metadata for org name={} id={}\n".format(org_metadata.org_name,
+                                                                                                  org_id))
+        self.transact_contract_command("Registry", "changeOrganizationMetadataURI", params)
+        self._printout("Organization updated successfully with id:\n%s" % org_id)
 
     def list_services(self):
         org_id = self.args.org_id

snet/cli/commands/mpe_channel.py

@@ -4,6 +4,7 @@
 import shutil
 import tempfile
 from collections import defaultdict
+from importlib.metadata import metadata
 from pathlib import Path
 
 from eth_abi.codec import ABICodec
@@ -15,9 +16,9 @@
 from snet.cli.metadata.service import mpe_service_metadata_from_json, load_mpe_service_metadata
 from snet.cli.metadata.organization import OrganizationMetadata
 from snet.cli.utils.agix2cogs import cogs2stragix
-from snet.cli.utils.ipfs_utils import bytesuri_to_hash, get_from_ipfs_and_checkhash, safe_extract_proto_from_ipfs
+from snet.cli.utils.ipfs_utils import get_from_ipfs_and_checkhash
 from snet.cli.utils.utils import abi_decode_struct_to_dict, abi_get_element_by_name, \
-    compile_proto, type_converter
+    compile_proto, type_converter, bytesuri_to_hash, get_file_from_filecoin, download_and_safe_extract_proto
 
 
 # we inherit MPEServiceCommand because we need _get_service_metadata_from_registry
@@ -577,12 +578,14 @@ def _get_service_registration(self):
 
     def _get_service_metadata_from_registry(self):
         response = self._get_service_registration()
-        metadata_hash = bytesuri_to_hash(response["metadataURI"])
-        metadata = get_from_ipfs_and_checkhash(
-            self._get_ipfs_client(), metadata_hash)
-        metadata = metadata.decode("utf-8")
-        metadata = mpe_service_metadata_from_json(metadata)
-        return metadata
+        storage_type, metadata_hash = bytesuri_to_hash(response["metadataURI"])
+        if storage_type == "ipfs":
+            service_metadata = get_from_ipfs_and_checkhash(self._get_ipfs_client(), metadata_hash)
+        else:
+            service_metadata = get_file_from_filecoin(metadata_hash)
+        service_metadata = service_metadata.decode("utf-8")
+        service_metadata = mpe_service_metadata_from_json(service_metadata)
+        return service_metadata
 
     def _init_or_update_service_if_needed(self, metadata, service_registration):
         # if service was already initialized and metadataURI hasn't changed we do nothing
@@ -609,8 +612,8 @@ def _init_or_update_service_if_needed(self, metadata, service_registration):
         try:
             spec_dir = os.path.join(service_dir, "service_spec")
             os.makedirs(spec_dir, mode=0o700)
-            safe_extract_proto_from_ipfs(
-                self._get_ipfs_client(), metadata["model_ipfs_hash"], spec_dir)
+            service_api_source = metadata.get("service_api_source") or metadata.get("model_ipfs_hash")
+            download_and_safe_extract_proto(service_api_source, spec_dir, self._get_ipfs_client())
 
             # compile .proto files
             if not compile_proto(Path(spec_dir), service_dir):