diff --git a/.gitignore b/.gitignore index 2a45881..92e7a68 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ log.txt test.* # Ignore the IDE gitignore files .idea/ +# Test cases generate the following +data/ +Config.yaml diff --git a/Abstrations/files.go b/Abstrations/files.go new file mode 100644 index 0000000..cd81019 --- /dev/null +++ b/Abstrations/files.go @@ -0,0 +1,107 @@ +/* +File Name: abstractions.go +Copyright: 2021 Peernet s.r.o. +Authors: Peter Kleissner, Akilan Selvacoumar +*/ +package Abstrations + +import ( + "encoding/hex" + "errors" + "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/warehouse" + "github.com/PeernetOfficial/core/webapi" + "github.com/google/uuid" + "path/filepath" + "time" +) + +/* +Library description +to about abstracted function to easily add and remove files. +*/ + +type TouchReturn struct { + BlockchainHeight uint64 + BlockchainVersion uint64 +} + +// Touch abstracted function that creates a file +// and adds the file to the warehouse and +// blockchain +// returns blockchain version and height +func Touch(b *core.Backend, filePath string) (*TouchReturn, error) { + // Creates a File in the warehouse + hash, _, err := b.UserWarehouse.CreateFileFromPath(filePath) + if err != nil { + return nil, err + } + + // Add the File to the local blockchain + var input webapi.ApiBlockAddFiles + var inputFiles []webapi.ApiFile + var inputFile webapi.ApiFile + + // Write File information to the input File + inputFile.Date = time.Now() + // Folder and File name + dir, file := filepath.Split(filePath) + inputFile.Folder = dir + inputFile.Name = file + inputFile.ID = uuid.New() + inputFile.Hash = hash + + // Get the public key of the current node + _, publicKey := b.ExportPrivateKey() + inputFile.NodeID = []byte(hex.EncodeToString(publicKey.SerializeCompressed())) + + inputFiles = append(inputFiles, inputFile) + + input.Files = inputFiles + + var filesAdd []blockchain.BlockRecordFile + + for _, File := range input.Files { + if len(File.Hash) != protocol.HashSize { + return nil, errors.New("bad request") + } + if File.ID == uuid.Nil { // if the ID is not provided by the caller, set it + File.ID = uuid.New() + } + + // Verify that the File exists in the warehouse. Folders are exempt from this check as they are only virtual. + if !File.IsVirtualFolder() { + if _, err := warehouse.ValidateHash(File.Hash); err != nil { + return nil, errors.New("bad request when validating hash") + } else if _, fileInfo, status, _ := b.UserWarehouse.FileExists(File.Hash); status != warehouse.StatusOK { + //EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) + return nil, errors.New("file not in warehouse") + } else { + File.Size = fileInfo + } + } else { + File.Hash = protocol.HashData(nil) + File.Size = 0 + } + + blockRecord := webapi.BlockRecordFileFromAPI(File) + + // Set the merkle tree info as appropriate. + if !webapi.SetFileMerkleInfo(b, &blockRecord) { + return nil, errors.New("merkle information not set") + } + + filesAdd = append(filesAdd, blockRecord) + } + + newHeight, newVersion, _ := b.UserBlockchain.AddFiles(filesAdd) + + // Creating object for custom return type + var touchReturn TouchReturn + touchReturn.BlockchainHeight = newHeight + touchReturn.BlockchainVersion = newVersion + + return &touchReturn, nil +} diff --git a/Abstrations/files_test.go b/Abstrations/files_test.go new file mode 100644 index 0000000..9b809ce --- /dev/null +++ b/Abstrations/files_test.go @@ -0,0 +1,35 @@ +package Abstrations + +import ( + "fmt" + "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/webapi" + "github.com/google/uuid" + "testing" + "time" +) + +func InitPeernet() *core.Backend { + backendInit, status, err := core.Init("Your application/1.0", "Config.yaml", nil, nil) + if status != core.ExitSuccess { + fmt.Printf("Error %d initializing config: %s\n", status, err.Error()) + return nil + } + + // start config api server + webapi.Start(backendInit, []string{"127.0.0.1:5125"}, false, "", "", 10*time.Second, 10*time.Second, uuid.Nil) + + backendInit.Connect() + + return backendInit +} + +// Testing the touch function if the file is added or not +func TestBackend_Touch(t *testing.T) { + backend := InitPeernet() + touch, err := Touch(backend, "Config.yaml") + if err != nil { + t.Fail() + } + fmt.Printf("blockchain height: %v", touch.BlockchainHeight) +} diff --git a/webapi/Blockchain.go b/webapi/Blockchain.go index 0d6a362..28dd860 100644 --- a/webapi/Blockchain.go +++ b/webapi/Blockchain.go @@ -109,7 +109,7 @@ func (api *WebapiInstance) apiBlockchainRead(w http.ResponseWriter, r *http.Requ for _, record := range block.RecordsDecoded { switch v := record.(type) { case blockchain.BlockRecordFile: - result.RecordsDecoded = append(result.RecordsDecoded, blockRecordFileToAPI(v)) + result.RecordsDecoded = append(result.RecordsDecoded, BlockRecordFileToAPI(v)) case blockchain.BlockRecordProfile: result.RecordsDecoded = append(result.RecordsDecoded, blockRecordProfileToAPI(v)) diff --git a/webapi/Download.go b/webapi/Download.go index 0b6e766..b1ac98d 100644 --- a/webapi/Download.go +++ b/webapi/Download.go @@ -23,7 +23,7 @@ type apiResponseDownloadStatus struct { APIStatus int `json:"apistatus"` // Status of the API call. See DownloadResponseX. ID uuid.UUID `json:"id"` // Download ID. This can be used to query the latest status and take actions. DownloadStatus int `json:"downloadstatus"` // Status of the download. See DownloadX. - File apiFile `json:"file"` // File information. Only available for status >= DownloadWaitSwarm. + File ApiFile `json:"file"` // File information. Only available for status >= DownloadWaitSwarm. Progress struct { TotalSize uint64 `json:"totalsize"` // Total size in bytes. DownloadedSize uint64 `json:"downloadedsize"` // Count of bytes download so far. @@ -189,7 +189,7 @@ type downloadInfo struct { created time.Time // When the download was created. ended time.Time // When the download was finished (only status = DownloadFinished). - file apiFile // File metadata (only status >= DownloadWaitSwarm) + file ApiFile // File metadata (only status >= DownloadWaitSwarm) DiskFile struct { // Target file on disk to store downloaded data Name string // File name diff --git a/webapi/File.go b/webapi/File.go index a7da52f..27928b6 100644 --- a/webapi/File.go +++ b/webapi/File.go @@ -18,8 +18,8 @@ import ( "github.com/google/uuid" ) -// apiFileMetadata contains metadata information. -type apiFileMetadata struct { +// ApiFileMetadata contains metadata information. +type ApiFileMetadata struct { Type uint16 `json:"type"` // See core.TagX constants. Name string `json:"name"` // User friendly name of the metadata type. Use the Type fields to identify the metadata as this name may change. // Depending on the exact type, one of the below fields is used for proper encoding: @@ -29,8 +29,8 @@ type apiFileMetadata struct { Number uint64 `json:"number"` // Number } -// apiFile is the metadata of a file published on the blockchain -type apiFile struct { +// ApiFile is the metadata of a file published on the blockchain +type ApiFile struct { ID uuid.UUID `json:"id"` // Unique ID. Hash []byte `json:"hash"` // Blake3 hash of the file data Type uint8 `json:"type"` // File Type. For example audio or document. See TypeX. @@ -41,13 +41,13 @@ type apiFile struct { Description string `json:"description"` // Description. This is expected to be multiline and contain hashtags! Date time.Time `json:"date"` // Date shared NodeID []byte `json:"nodeid"` // Node ID, owner of the file. Read only. - Metadata []apiFileMetadata `json:"metadata"` // Additional metadata. + Metadata []ApiFileMetadata `json:"metadata"` // Additional metadata. } // --- conversion from core to API data --- -func blockRecordFileToAPI(input blockchain.BlockRecordFile) (output apiFile) { - output = apiFile{ID: input.ID, Hash: input.Hash, NodeID: input.NodeID, Type: input.Type, Format: input.Format, Size: input.Size, Metadata: []apiFileMetadata{}} +func BlockRecordFileToAPI(input blockchain.BlockRecordFile) (output ApiFile) { + output = ApiFile{ID: input.ID, Hash: input.Hash, NodeID: input.NodeID, Type: input.Type, Format: input.Format, Size: input.Size, Metadata: []ApiFileMetadata{}} for _, tag := range input.Tags { switch tag.Type { @@ -65,23 +65,23 @@ func blockRecordFileToAPI(input blockchain.BlockRecordFile) (output apiFile) { case blockchain.TagDateCreated: date, _ := tag.Date() - output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Date Created", Date: date}) + output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Name: "Date Created", Date: date}) case blockchain.TagSharedByCount: - output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Shared By Count", Number: tag.Number()}) + output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Name: "Shared By Count", Number: tag.Number()}) case blockchain.TagSharedByGeoIP: - output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Shared By GeoIP", Text: tag.Text()}) + output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Name: "Shared By GeoIP", Text: tag.Text()}) default: - output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Blob: tag.Data}) + output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Blob: tag.Data}) } } return output } -func blockRecordFileFromAPI(input apiFile) (output blockchain.BlockRecordFile) { +func BlockRecordFileFromAPI(input ApiFile) (output blockchain.BlockRecordFile) { output = blockchain.BlockRecordFile{ID: input.ID, Hash: input.Hash, Type: input.Type, Format: input.Format, Size: input.Size} if input.Name != "" { @@ -115,9 +115,9 @@ func blockRecordFileFromAPI(input apiFile) (output blockchain.BlockRecordFile) { // --- File API --- -// apiBlockAddFiles contains a list of files from the blockchain -type apiBlockAddFiles struct { - Files []apiFile `json:"files"` // List of files +// ApiBlockAddFiles contains a list of files from the blockchain +type ApiBlockAddFiles struct { + Files []ApiFile `json:"files"` // List of files Status int `json:"status"` // Status of the operation, only used when this structure is returned from the API. } @@ -128,12 +128,12 @@ If any file is not stored in the Warehouse, the function aborts with the status If the block record encoding fails for any file, this function aborts with the status code StatusCorruptBlockRecord. In case the function aborts, the blockchain remains unchanged. -Request: POST /blockchain/file/add with JSON structure apiBlockAddFiles +Request: POST /blockchain/file/add with JSON structure ApiBlockAddFiles Response: 200 with JSON structure apiBlockchainBlockStatus 400 if invalid input */ func (api *WebapiInstance) apiBlockchainFileAdd(w http.ResponseWriter, r *http.Request) { - var input apiBlockAddFiles + var input ApiBlockAddFiles if err := DecodeJSON(w, r, &input); err != nil { return } @@ -165,10 +165,10 @@ func (api *WebapiInstance) apiBlockchainFileAdd(w http.ResponseWriter, r *http.R file.Size = 0 } - blockRecord := blockRecordFileFromAPI(file) + blockRecord := BlockRecordFileFromAPI(file) // Set the merkle tree info as appropriate. - if !setFileMerkleInfo(api.backend, &blockRecord) { + if !SetFileMerkleInfo(api.backend, &blockRecord) { EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) return } @@ -185,15 +185,15 @@ func (api *WebapiInstance) apiBlockchainFileAdd(w http.ResponseWriter, r *http.R apiBlockchainFileList lists all files stored on the blockchain. Request: GET /blockchain/file/list -Response: 200 with JSON structure apiBlockAddFiles +Response: 200 with JSON structure ApiBlockAddFiles */ func (api *WebapiInstance) apiBlockchainFileList(w http.ResponseWriter, r *http.Request) { files, status := api.backend.UserBlockchain.ListFiles() - var result apiBlockAddFiles + var result ApiBlockAddFiles for _, file := range files { - result.Files = append(result.Files, blockRecordFileToAPI(file)) + result.Files = append(result.Files, BlockRecordFileToAPI(file)) } result.Status = status @@ -205,11 +205,11 @@ func (api *WebapiInstance) apiBlockchainFileList(w http.ResponseWriter, r *http. apiBlockchainFileDelete deletes files with the provided IDs. Other fields are ignored. It will automatically delete the file in the Warehouse if there are no other references. -Request: POST /blockchain/file/delete with JSON structure apiBlockAddFiles +Request: POST /blockchain/file/delete with JSON structure ApiBlockAddFiles Response: 200 with JSON structure apiBlockchainBlockStatus */ func (api *WebapiInstance) apiBlockchainFileDelete(w http.ResponseWriter, r *http.Request) { - var input apiBlockAddFiles + var input ApiBlockAddFiles if err := DecodeJSON(w, r, &input); err != nil { return } @@ -237,12 +237,12 @@ func (api *WebapiInstance) apiBlockchainFileDelete(w http.ResponseWriter, r *htt /* apiBlockchainSelfUpdateFile updates files that are already published on the blockchain. -Request: POST /blockchain/file/update with JSON structure apiBlockAddFiles +Request: POST /blockchain/file/update with JSON structure ApiBlockAddFiles Response: 200 with JSON structure apiBlockchainBlockStatus 400 if invalid input */ func (api *WebapiInstance) apiBlockchainFileUpdate(w http.ResponseWriter, r *http.Request) { - var input apiBlockAddFiles + var input ApiBlockAddFiles if err := DecodeJSON(w, r, &input); err != nil { return } @@ -274,10 +274,10 @@ func (api *WebapiInstance) apiBlockchainFileUpdate(w http.ResponseWriter, r *htt file.Size = 0 } - blockRecord := blockRecordFileFromAPI(file) + blockRecord := BlockRecordFileFromAPI(file) // Set the merkle tree info as appropriate. - if !setFileMerkleInfo(api.backend, &blockRecord) { + if !SetFileMerkleInfo(api.backend, &blockRecord) { EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) return } @@ -293,7 +293,7 @@ func (api *WebapiInstance) apiBlockchainFileUpdate(w http.ResponseWriter, r *htt // ---- metadata functions ---- // GetMetadata returns the specified metadata or nil if not available. -func (file *apiFile) GetMetadata(Type uint16) (info *apiFileMetadata) { +func (file *ApiFile) GetMetadata(Type uint16) (info *ApiFileMetadata) { for n := range file.Metadata { if file.Metadata[n].Type == Type { return &file.Metadata[n] @@ -304,7 +304,7 @@ func (file *apiFile) GetMetadata(Type uint16) (info *apiFileMetadata) { } // GetNumber returns the data as number. 0 if not available. -func (info *apiFileMetadata) GetNumber() uint64 { +func (info *ApiFileMetadata) GetNumber() uint64 { if info == nil { return 0 } @@ -313,12 +313,12 @@ func (info *apiFileMetadata) GetNumber() uint64 { } // IsVirtualFolder returns true if the file is a virtual folder -func (file *apiFile) IsVirtualFolder() bool { +func (file *ApiFile) IsVirtualFolder() bool { return file.Type == core.TypeFolder && file.Format == core.FormatFolder } -// setFileMerkleInfo sets the merkle fields in the BlockRecordFile -func setFileMerkleInfo(backend *core.Backend, file *blockchain.BlockRecordFile) (valid bool) { +// SetFileMerkleInfo sets the merkle fields in the BlockRecordFile +func SetFileMerkleInfo(backend *core.Backend, file *blockchain.BlockRecordFile) (valid bool) { if file.Size <= merkle.MinimumFragmentSize { // If smaller or equal than the minimum fragment size, the merkle tree is not used. file.MerkleRootHash = file.Hash diff --git a/webapi/Search Dispatch.go b/webapi/Search Dispatch.go index 3107bfa..7044773 100644 --- a/webapi/Search Dispatch.go +++ b/webapi/Search Dispatch.go @@ -68,7 +68,7 @@ resultLoop: } // new result - newFile := blockRecordFileToAPI(file) + newFile := BlockRecordFileToAPI(file) job.Files = append(job.Files, &newFile) job.AllFiles = append(job.AllFiles, &newFile) diff --git a/webapi/Search Job.go b/webapi/Search Job.go index ffdf5cf..5b4270f 100644 --- a/webapi/Search Job.go +++ b/webapi/Search Job.go @@ -56,14 +56,14 @@ type SearchJob struct { clientsMutex sync.Mutex // mutex for manipulating client list // List of files found but not yet returned via API to the caller. They are subject to sorting. - Files []*apiFile + Files []*ApiFile requireSort bool // if Files requires sort before returning the results // FreezeFiles is a list of files that were already finally delivered via the API. They may NOT change in sorting. - FreezeFiles []*apiFile + FreezeFiles []*ApiFile // List of all files. Does not change based on sorting or runtime filters. This list only gets expanded. - AllFiles []*apiFile + AllFiles []*ApiFile ResultSync sync.Mutex // ResultSync ensures unique access to the file results @@ -101,7 +101,7 @@ func (api *WebapiInstance) CreateSearchJob(Timeout time.Duration, MaxResults int } // ReturnResult returns the selected results. -func (job *SearchJob) ReturnResult(Offset, Limit int) (Result []*apiFile) { +func (job *SearchJob) ReturnResult(Offset, Limit int) (Result []*ApiFile) { if Limit == 0 { return Result } @@ -159,7 +159,7 @@ func (job *SearchJob) ReturnResult(Offset, Limit int) (Result []*apiFile) { } // ReturnNext returns the next results. Call must be serialized. -func (job *SearchJob) ReturnNext(Limit int) (Result []*apiFile) { +func (job *SearchJob) ReturnNext(Limit int) (Result []*ApiFile) { Result = job.ReturnResult(job.currentOffset, Limit) job.currentOffset += len(Result) @@ -167,7 +167,7 @@ func (job *SearchJob) ReturnNext(Limit int) (Result []*apiFile) { } // PeekResult returns the selected results but will not change any frozen files or impact auto offset -func (job *SearchJob) PeekResult(Offset, Limit int) (Result []*apiFile) { +func (job *SearchJob) PeekResult(Offset, Limit int) (Result []*ApiFile) { job.ResultSync.Lock() defer job.ResultSync.Unlock() @@ -218,7 +218,7 @@ func (job *SearchJob) RuntimeFilter(Filter SearchFilter) { job.currentOffset = 0 // files remain in AllFiles, but Files needs to be filtered based on the new filter - job.Files = []*apiFile{} + job.Files = []*ApiFile{} job.requireSort = false // Sorting is done immediately below // set Files based on AllFiles with the filter @@ -236,7 +236,7 @@ func (job *SearchJob) RuntimeFilter(Filter SearchFilter) { } // isFileFiltered returns true if the file conforms to the runtime filter. If there is no runtime filter, it always returns true. -func (job *SearchJob) isFileFiltered(file *apiFile) bool { +func (job *SearchJob) isFileFiltered(file *ApiFile) bool { if job.filtersRuntime.FileType >= 0 && file.Type != uint8(job.filtersRuntime.FileType) { return false } @@ -258,7 +258,7 @@ func (job *SearchJob) isFileFiltered(file *apiFile) bool { } // SortFiles sorts a list of files. It returns a sorted list. 0 = no sorting, 1 = Relevance ASC, 2 = Relevance DESC, 3 = Date ASC, 4 = Date DESC, 5 = Name ASC, 6 = Name DESC -func SortFiles(files []*apiFile, Sort int) (sorted []*apiFile) { +func SortFiles(files []*ApiFile, Sort int) (sorted []*ApiFile) { switch Sort { case SortRelevanceAsc: sort.SliceStable(files, func(i, j int) bool { return files[i].Date.Before(files[j].Date) }) // first as date for secondary sorting @@ -424,7 +424,7 @@ func (job *SearchJob) Statistics() (result SearchStatisticData) { } // statsAdd counts the files in the statistics -func (job *SearchJob) statsAdd(files ...*apiFile) { +func (job *SearchJob) statsAdd(files ...*ApiFile) { job.stats.Lock() defer job.stats.Unlock() diff --git a/webapi/Search.go b/webapi/Search.go index 0e61360..e75f321 100644 --- a/webapi/Search.go +++ b/webapi/Search.go @@ -62,7 +62,7 @@ type SearchRequestResponse struct { // SearchResult contains the search results. type SearchResult struct { Status int `json:"status"` // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying - Files []apiFile `json:"files"` // List of files found + Files []ApiFile `json:"files"` // List of files found Statistic interface{} `json:"statistic"` // Statistics of all results (independent from applied filters), if requested. Only set if files are returned (= if statistics changed). See SearchStatisticData. } @@ -161,7 +161,7 @@ func (api *WebapiInstance) apiSearchResult(w http.ResponseWriter, r *http.Reques } // query all results - var resultFiles []*apiFile + var resultFiles []*ApiFile if errOffset == nil { resultFiles = job.ReturnResult(offset, limit) } else { @@ -169,7 +169,7 @@ func (api *WebapiInstance) apiSearchResult(w http.ResponseWriter, r *http.Reques } var result SearchResult - result.Files = []apiFile{} + result.Files = []ApiFile{} // loop over results for n := range resultFiles { @@ -241,7 +241,7 @@ func (api *WebapiInstance) apiSearchResultStream(w http.ResponseWriter, r *http. // Only exit if limit is reached if used, otherwise only if there are no result or the connection breaks. for { // query all results - var resultFiles []*apiFile + var resultFiles []*ApiFile queryCount := 1 if useLimit { @@ -255,7 +255,7 @@ func (api *WebapiInstance) apiSearchResultStream(w http.ResponseWriter, r *http. // loop over results var result SearchResult - result.Files = []apiFile{} + result.Files = []ApiFile{} for n := range resultFiles { result.Files = append(result.Files, *resultFiles[n]) @@ -366,11 +366,11 @@ func (api *WebapiInstance) apiExplore(w http.ResponseWriter, r *http.Request) { resultFiles := api.queryRecentShared(api.backend, fileType, uint64(limit*20/100), uint64(limit)) var result SearchResult - result.Files = []apiFile{} + result.Files = []ApiFile{} // loop over results for n := range resultFiles { - result.Files = append(result.Files, blockRecordFileToAPI(resultFiles[n])) + result.Files = append(result.Files, BlockRecordFileToAPI(resultFiles[n])) } if len(result.Files) == 0 {