diff --git a/.github/workflows/pr-testing.yml b/.github/workflows/pr-testing.yml new file mode 100644 index 000000000..f712a3216 --- /dev/null +++ b/.github/workflows/pr-testing.yml @@ -0,0 +1,33 @@ +name: PR testing + +on: + pull_request: + branches: [ master ] + +jobs: + mavenTesting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Set up the Java JDK + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + + - name: Run all tests + run: | + mvn -B clean test -DskipTests=false --file pom.xml + if [ -f "target/site/jacoco/index.html" ]; then echo "Total coverage: $(cat target/site/jacoco/index.html | grep -o 'Total[^%]*%' | grep -o '[0-9]*%')"; fi + + - name: Log coverage percentage + run: | + if [ ! -f "target/site/jacoco/index.html" ]; then echo "No coverage information available"; fi + if [ -f "target/site/jacoco/index.html" ]; then echo "Total coverage: $(cat target/site/jacoco/index.html | grep -o 'Total[^%]*%' | grep -o '[0-9]*%')"; fi diff --git a/.gitignore b/.gitignore index 88056ba09..e26d62449 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ /db* +/lists/ /bin/ /target/ +/qortal-backup/ /log.txt.* /arbitrary* /Qortal-BTC* @@ -14,5 +16,18 @@ /settings.json /testnet* /settings*.json -/testchain.json -/run-testnet.sh +/testchain*.json +/run-testnet*.sh +/.idea +/qortal.iml +.DS_Store +/src/main/resources/resources +/*.jar +/run.pid +/run.log +/WindowsInstaller/Install Files/qortal.jar +/*.7z +/tmp +/data* +/src/test/resources/arbitrary/*/.qortal/cache +apikey.txt diff --git a/AutoUpdates.md b/AutoUpdates.md index ccb441ca3..7f2482467 100644 --- a/AutoUpdates.md +++ b/AutoUpdates.md @@ -1,5 +1,20 @@ # Auto Updates +## TL;DR: how-to + +* Prepare new release version (see way below for details) +* Assuming you are in git 'master' branch, at HEAD +* Shutdown local node if running +* Build auto-update download: `tools/build-auto-update.sh` - uploads auto-update file into new git branch +* Restart local node +* Publish auto-update transaction using *private key* for **non-admin** member of "dev" group: + `tools/publish-auto-update.sh non-admin-dev-member-private-key-in-base58` +* Wait for auto-update `ARBITRARY` transaction to be confirmed into a block +* Have "dev" group admins 'approve' auto-update using `tools/approve-auto-update.sh` + This tool will prompt for *private key* of **admin** of "dev" group +* A minimum number of admins are required for approval, and a minimum number of blocks must pass also. +* Nodes will start to download, and apply, the update over the next 20 minutes or so (see CHECK_INTERVAL in AutoUpdate.java) + ## Theory * Using a specific git commit (e.g. abcdef123) we produce a determinstic JAR with consistent hash. * To avoid issues with over-eager anti-virus / firewalls we obfuscate JAR using very simplistic XOR-based method. @@ -25,8 +40,8 @@ The same method is used to obfuscate and de-obfuscate: ## Typical download locations The git SHA1 commit hash is used to replace `%s` in various download locations, e.g.: -* https://github.com/QORT/qortal/raw/%s/qortal.update -* https://raw.githubusercontent.com@151.101.16.133/QORT/qortal/%s/qortal.update +* https://github.com/Qortal/qortal/raw/%s/qortal.update +* https://raw.githubusercontent.com@151.101.16.133/Qortal/qortal/%s/qortal.update These locations are part of the org.qortal.settings.Settings class and can be overriden in settings.json like: ``` @@ -45,4 +60,12 @@ $ java -cp qortal.jar org.qortal.XorUpdate usage: XorUpdate $ java -cp qortal.jar org.qortal.XorUpdate qortal.jar qortal.update $ -``` \ No newline at end of file +``` + +## Preparing new release version + +* Shutdown local node +* Modify `pom.xml` and increase version inside `` tag +* Commit new `pom.xml` and push to github, e.g. `git commit -m 'Bumped to v1.4.2' -- pom.xml; git push` +* Tag this new commit with same version: `git tag v1.4.2` +* Push tag up to github: `git push origin v1.4.2` diff --git a/DATABASE.md b/DATABASE.md index dc4eb6f1b..8c53c6403 100644 --- a/DATABASE.md +++ b/DATABASE.md @@ -4,10 +4,10 @@ You can examine your node's database using [HSQLDB's "sqltool"](http://www.hsqld It's a good idea to install "rlwrap" (ReadLine wrapper) too as sqltool doesn't support command history/editing. Typical command line for sqltool would be: -`rlwrap java -cp ${HSQLDB_JAR}:${SQLTOOL_JAR} org.hsqldb.cmdline.SqlTool --rcFile=${SQLTOOL_RC} qora` +`rlwrap java -cp ${HSQLDB_JAR}:${SQLTOOL_JAR} org.hsqldb.cmdline.SqlTool --rcFile=${SQLTOOL_RC} qortal` `${HSQLDB_JAR}` should be set with pathname where Maven downloaded hsqldb, -typically `${HOME}/.m2/repository/org/hsqldb/hsqldb/2.5.0/hsqldb-2.5.0.jar` +typically `${HOME}/.m2/repository/org/hsqldb/hsqldb/2.5.1/hsqldb-2.5.1.jar` `${SQLTOOL_JAR}` should be set with pathname where Maven downloaded sqltool, typically `${HOME}/.m2/repository/org/hsqldb/sqltool/2.5.0/sqltool-2.5.0.jar` @@ -25,10 +25,16 @@ Above `url` component `file:db/blockchain` assumes you will call `sqltool` from Another idea is to assign a shell alias in your `.bashrc` like: ``` -export HSQLDB_JAR=${HOME}/.m2/repository/org/hsqldb/hsqldb/2.5.0/hsqldb-2.5.0.jar +export HSQLDB_JAR=${HOME}/.m2/repository/org/hsqldb/hsqldb/2.5.1/hsqldb-2.5.1.jar export SQLTOOL_JAR=${HOME}/.m2/repository/org/hsqldb/sqltool/2.5.0/sqltool-2.5.0.jar alias sqltool='rlwrap java -cp ${HSQLDB_JAR}:${SQLTOOL_JAR} org.hsqldb.cmdline.SqlTool --rcFile=${SQLTOOL_RC}' ``` So you can simply type: `sqltool qortal` Don't forget to use `SHUTDOWN;` before exiting sqltool so that database files are closed cleanly. + +## Quick and dirty version + +With `sqltool-2.5.0.jar` and `qortal.jar` in current directory, and database in `db/` + +`java -cp qortal.jar:sqltool-2.5.0.jar org.hsqldb.cmdline.SqlTool --inlineRc=url=jdbc:hsqldb:file:db/blockchain` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..b06f7659c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM maven:3-openjdk-11 as builder + +WORKDIR /work +COPY ./ /work/ +RUN mvn clean package + +### +FROM openjdk:11 + +RUN useradd -r -u 1000 -g users qortal && \ + mkdir /usr/local/qortal /qortal && \ + chown 1000:100 /qortal + +COPY --from=builder /work/log4j2.properties /usr/local/qortal/ +COPY --from=builder /work/target/qortal*.jar /usr/local/qortal/qortal.jar + +USER 1000:100 + +EXPOSE 12391 12392 +HEALTHCHECK --start-period=5m CMD curl -sf http://127.0.0.1:12391/admin/info || exit 1 + +WORKDIR /qortal +VOLUME /qortal + +ENTRYPOINT ["java"] +CMD ["-Djava.net.preferIPv4Stack=false", "-jar", "/usr/local/qortal/qortal.jar"] diff --git a/README.md b/README.md index f11a161cb..9dd9ad60b 100644 --- a/README.md +++ b/README.md @@ -9,4 +9,4 @@ - Create basic *settings.json* file: `echo '{}' > settings.json` - Run JAR in same working directory as *settings.json*: `java -jar target/qortal-1.0.jar` - Wrap in shell script, add JVM flags, redirection, backgrounding, etc. as necessary. -- Or use supplied example shell script: *run.sh* +- Or use supplied example shell script: *start.sh* diff --git a/TestNets.md b/TestNets.md new file mode 100644 index 000000000..6f8e92e69 --- /dev/null +++ b/TestNets.md @@ -0,0 +1,69 @@ +# How to build a testnet + +## Create testnet blockchain config + +- You can begin by copying the mainnet blockchain config `src/main/resources/blockchain.json` +- Insert `"isTestChain": true,` after the opening `{` +- Modify testnet genesis block + +### Testnet genesis block + +- Set `timestamp` to a nearby future value, e.g. 15 mins from 'now' + This is to give yourself enough time to set up other testnet nodes +- Retain the initial `ISSUE_ASSET` transactions! +- Add `ACCOUNT_FLAGS` transactions with `"andMask": -1, "orMask": 1, "xorMask": 0` to create founders +- Add at least one `REWARD_SHARE` transaction otherwise no-one can mint initial blocks! + You will need to calculate `rewardSharePublicKey` (and private key), + or make a new account on mainnet and use self-share key values +- Add `ACCOUNT_LEVEL` transactions to set initial level of accounts as needed +- Add `GENESIS` transactions to add QORT/LEGACY_QORA funds to accounts as needed + +## Testnet `settings.json` + +- Create a new `settings-test.json` +- Make sure to add `"isTestNet": true,` +- Make sure to reference testnet blockchain config file: `"blockchainConfig": "testchain.json",` +- It is a good idea to use a separate database: `"repositoryPath": "db-testnet",` +- You might also need to add `"bitcoinNet": "TEST3",` and `"litecoinNet": "TEST3",` + +## Other nodes + +- Copy `testchain.json` and `settings-test.json` to other nodes +- Alternatively, you can run multiple nodes on the same machine by: + * Copying `settings-test.json` to `settings-test-1.json` + * Configure different `repositoryPath` + * Configure use of different ports: + + `"apiPort": 22391,` + + `"listenPort": 22392,` + +## Starting-up + +- Start up at least as many nodes as `minBlockchainPeers` (or adjust this value instead) +- Probably best to perform API call `DELETE /peers/known` +- Add other nodes via API call `POST /peers ` +- Add minting private key to node(s) via API call `POST /admin/mintingaccounts ` + This key must have corresponding `REWARD_SHARE` transaction in testnet genesis block +- Wait for genesis block timestamp to pass +- A node should mint block 2 approximately 60 seconds after genesis block timestamp +- Other testnet nodes will sync *as long as there is at least `minBlockchainPeers` peers with an "up-to-date" chain` +- You can also use API call `POST /admin/forcesync ` on stuck nodes + +## Dealing with stuck chain + +Maybe your nodes have been offline and no-one has minted a recent testnet block. +Your options are: + +- Start a new testnet from scratch +- Fire up your testnet node(s) +- Force one of your nodes to mint by: + + Set a debugger breakpoint on Settings.getMinBlockchainPeers() + + When breakpoint is hit, change `this.minBlockchainPeers` to zero, then continue +- Once one of your nodes has minted blocks up to 'now', you can use "forcesync" on the other nodes + +## Tools + +- `qort` tool, but use `-t` option for default testnet API port (62391) +- `qort` tool, but first set shell variable: `export BASE_URL=some-node-hostname-or-ip:port` +- `qort` tool, but prepend with one-time shell variable: `BASE_URL=some-node-hostname-or-ip:port qort ......` +- `peer-heights`, but use `-t` option, or `BASE_URL` shell variable as above + diff --git a/WindowsInstaller/Install Files/AppData/settings.json b/WindowsInstaller/Install Files/AppData/settings.json new file mode 100755 index 000000000..088afef42 --- /dev/null +++ b/WindowsInstaller/Install Files/AppData/settings.json @@ -0,0 +1,3 @@ +{ + "apiDocumentationEnabled": true +} diff --git a/WindowsInstaller/Install Files/log4j2.properties b/WindowsInstaller/Install Files/log4j2.properties new file mode 100755 index 000000000..44e1b1e3e --- /dev/null +++ b/WindowsInstaller/Install Files/log4j2.properties @@ -0,0 +1,70 @@ +rootLogger.level = info +# On Windows, uncomment next line to set dirname: +# property.dirname = ${sys:user.home}\\AppData\\Local\\qortal\\ +property.filename = ${sys:log4j2.filenameTemplate:-log.txt} + +rootLogger.appenderRef.console.ref = stdout +rootLogger.appenderRef.rolling.ref = FILE + +# Suppress extraneous bitcoinj library output +logger.bitcoinj.name = org.bitcoinj +logger.bitcoinj.level = error + +# Override HSQLDB logging level to "warn" as too much is logged at "info" +logger.hsqldb.name = hsqldb.db +logger.hsqldb.level = warn + +# Support optional, per-session HSQLDB debugging +logger.hsqldbRepository.name = org.qortal.repository.hsqldb +logger.hsqldbRepository.level = debug + +# Suppress extraneous Jersey warning +logger.jerseyInject.name = org.glassfish.jersey.internal.inject.Providers +logger.jerseyInject.level = off + +# Suppress extraneous Jersey EOF 'errors' (actually remote peers disconnecting early) +logger.jerseyEOF.name = org.glassfish.jersey.server.internal +logger.jerseyEOF.level = off + +# Suppress extraneous Jetty entries +# 2019-02-14 11:46:27 INFO ContextHandler:851 - Started o.e.j.s.ServletContextHandler@6949e948{/,null,AVAILABLE} +# 2019-02-14 11:46:27 INFO AbstractConnector:289 - Started ServerConnector@50ad322b{HTTP/1.1,[http/1.1]}{0.0.0.0:9085} +# 2019-02-14 11:46:27 INFO Server:374 - jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_181-b13 +# 2019-02-14 11:46:27 INFO Server:411 - Started @2539ms +logger.jetty.name = org.eclipse.jetty +logger.jetty.level = warn +# Even more extraneous Jetty output +# 2019-01-26 02:18:10 WARN ResourceService:718 - java.util.concurrent.TimeoutException: Idle timeout expired: 30000/30000 ms +logger.jettyRS.name = org.eclipse.jetty.server.ResourceService +logger.jettyRS.level = error + +# Suppress extraneous slf4j entries +# 2019-02-14 11:46:27 INFO log:193 - Logging initialized @1636ms to org.eclipse.jetty.util.log.Slf4jLog +logger.slf4j.name = org.slf4j +logger.slf4j.level = warn + +# Suppress extraneous Reflections entry +# 2019-02-27 10:45:25 WARN Reflections:179 - given scan urls are empty. set urls in the configuration +logger.orgReflections.name = org.reflections.Reflections +logger.orgReflections.level = off +logger.sunReflections.name = sun.reflect.Reflection +logger.sunReflections.level = off + +appender.console.type = Console +appender.console.name = stdout +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +appender.console.filter.threshold.type = ThresholdFilter +appender.console.filter.threshold.level = error + +appender.rolling.type = RollingFile +appender.rolling.name = FILE +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +appender.rolling.filePattern = ./${filename}.%i +appender.rolling.policy.type = SizeBasedTriggeringPolicy +appender.rolling.policy.size = 4MB +# Set the immediate flush to true (default) +# appender.rolling.immediateFlush = true +# Set the append to true (default), should not overwrite +# appender.rolling.append=true diff --git a/WindowsInstaller/Install Files/ntpcfg.bat b/WindowsInstaller/Install Files/ntpcfg.bat new file mode 100755 index 000000000..e9725808f --- /dev/null +++ b/WindowsInstaller/Install Files/ntpcfg.bat @@ -0,0 +1,33 @@ +@echo off + +:: BatchGotAdmin +:------------------------------------- +REM --> Check for permissions +>nul 2>&1 "%SYSTEMROOT%\system32\cacls.exe" "%SYSTEMROOT%\system32\config\system" + +REM --> If error flag set, we do not have admin. +if '%errorlevel%' NEQ '0' ( + echo Requesting administrative privileges... + goto UACPrompt +) else ( goto gotAdmin ) + +:UACPrompt + echo Set UAC = CreateObject^("Shell.Application"^) > "%temp%\getadmin.vbs" + echo UAC.ShellExecute "%~s0", "", "", "runas", 1 >> "%temp%\getadmin.vbs" + + "%temp%\getadmin.vbs" + exit /B + +:gotAdmin + if exist "%temp%\getadmin.vbs" ( del "%temp%\getadmin.vbs" ) + pushd "%CD%" + CD /D "%~dp0" +:-------------------------------------- + +net stop "Windows Time" + +w32tm /config "/manualpeerlist:pool.ntp.org 0.pool.ntp.org 1.pool.ntp.org 2.pool.ntp.org 3.pool.ntp.org cn.pool.ntp.org 0.cn.pool.ntp.org 1.cn.pool.ntp.org 2.cn.pool.ntp.org 3.cn.pool.ntp.org" + +net start "Windows Time" + +sc config w32time start= auto diff --git a/WindowsInstaller/Qortal.aip b/WindowsInstaller/Qortal.aip new file mode 100755 index 000000000..f69f0682f --- /dev/null +++ b/WindowsInstaller/Qortal.aip @@ -0,0 +1,1569 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/WindowsInstaller/README.md b/WindowsInstaller/README.md new file mode 100644 index 000000000..0a9f25228 --- /dev/null +++ b/WindowsInstaller/README.md @@ -0,0 +1,26 @@ +# Windows installer + +## Prerequisites + +* AdvancedInstaller v16 or better, and enterprise licence if translations are required +* Installed AdoptOpenJDK v11 64bit, full JDK *not* JRE + +## General build instructions + +If this is your first time opening the `qortal.aip` file then you might need to adjust +configured paths, or create a dummy `D:` drive with the expected layout. + +Typical build procedure: + +* Place the `qortal.jar` file in `Install-Files\` +* Open AdvancedInstaller with qortal.aip file +* If releasing a new version, change version number in: + + "Product Information" side menu + + "Product Details" side menu entry + + "Product Details" tab in "Product Details" pane + + "Product Version" entry box +* Click away to a different side menu entry, e.g. "Resources" -> "Files and Folders" +* You should be prompted whether to generate a new product key, click "Generate New" +* Click "Build" button +* New EXE should be generated in `Qortal-SetupFiles\` folder with correct version number + diff --git a/WindowsInstaller/dictionary.ail b/WindowsInstaller/dictionary.ail new file mode 100644 index 000000000..311702e1f --- /dev/null +++ b/WindowsInstaller/dictionary.ail @@ -0,0 +1,117 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/WindowsInstaller/qortal.ico b/WindowsInstaller/qortal.ico new file mode 100644 index 000000000..a44ed445d Binary files /dev/null and b/WindowsInstaller/qortal.ico differ diff --git a/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.jar b/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.jar new file mode 100644 index 000000000..10a92c071 Binary files /dev/null and b/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.jar differ diff --git a/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom b/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom new file mode 100644 index 000000000..96e0fe501 --- /dev/null +++ b/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom @@ -0,0 +1,9 @@ + + + 4.0.0 + com.dosse + WaifUPnP + 1.1 + POM was created from install:install-file + diff --git a/lib/com/dosse/WaifUPnP/maven-metadata-local.xml b/lib/com/dosse/WaifUPnP/maven-metadata-local.xml new file mode 100644 index 000000000..07d6ffd03 --- /dev/null +++ b/lib/com/dosse/WaifUPnP/maven-metadata-local.xml @@ -0,0 +1,12 @@ + + + com.dosse + WaifUPnP + + 1.1 + + 1.1 + + 20220218200127 + + diff --git a/lib/org/ciyam/AT/1.3.4/AT-1.3.4.jar b/lib/org/ciyam/AT/1.3.7/AT-1.3.7.jar similarity index 67% rename from lib/org/ciyam/AT/1.3.4/AT-1.3.4.jar rename to lib/org/ciyam/AT/1.3.7/AT-1.3.7.jar index 5abe2c77a..0e43ba18e 100644 Binary files a/lib/org/ciyam/AT/1.3.4/AT-1.3.4.jar and b/lib/org/ciyam/AT/1.3.7/AT-1.3.7.jar differ diff --git a/lib/org/ciyam/AT/1.3.4/AT-1.3.4.pom b/lib/org/ciyam/AT/1.3.7/AT-1.3.7.pom similarity index 94% rename from lib/org/ciyam/AT/1.3.4/AT-1.3.4.pom rename to lib/org/ciyam/AT/1.3.7/AT-1.3.7.pom index 39af6ac7d..a38b49f45 100644 --- a/lib/org/ciyam/AT/1.3.4/AT-1.3.4.pom +++ b/lib/org/ciyam/AT/1.3.7/AT-1.3.7.pom @@ -4,6 +4,6 @@ 4.0.0 org.ciyam AT - 1.3.4 + 1.3.7 POM was created from install:install-file diff --git a/lib/org/ciyam/AT/1.3.8/AT-1.3.8.jar b/lib/org/ciyam/AT/1.3.8/AT-1.3.8.jar new file mode 100644 index 000000000..5e7c36772 Binary files /dev/null and b/lib/org/ciyam/AT/1.3.8/AT-1.3.8.jar differ diff --git a/lib/org/ciyam/AT/1.3.8/AT-1.3.8.pom b/lib/org/ciyam/AT/1.3.8/AT-1.3.8.pom new file mode 100644 index 000000000..106adc389 --- /dev/null +++ b/lib/org/ciyam/AT/1.3.8/AT-1.3.8.pom @@ -0,0 +1,9 @@ + + + 4.0.0 + org.ciyam + AT + 1.3.8 + POM was created from install:install-file + diff --git a/lib/org/ciyam/AT/maven-metadata-local.xml b/lib/org/ciyam/AT/maven-metadata-local.xml index 2cf6d13a5..8f8b1f6ee 100644 --- a/lib/org/ciyam/AT/maven-metadata-local.xml +++ b/lib/org/ciyam/AT/maven-metadata-local.xml @@ -3,10 +3,14 @@ org.ciyam AT - 1.3.4 + 1.3.8 1.3.4 + 1.3.5 + 1.3.6 + 1.3.7 + 1.3.8 - 20200414162728 + 20200925114415 diff --git a/log4j2.properties b/log4j2.properties index fdbf51ddf..44e1b1e3e 100644 --- a/log4j2.properties +++ b/log4j2.properties @@ -61,7 +61,7 @@ appender.rolling.type = RollingFile appender.rolling.name = FILE appender.rolling.layout.type = PatternLayout appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -appender.rolling.filePattern = ${dirname:-}${filename}.%i +appender.rolling.filePattern = ./${filename}.%i appender.rolling.policy.type = SizeBasedTriggeringPolicy appender.rolling.policy.size = 4MB # Set the immediate flush to true (default) diff --git a/pom.xml b/pom.xml index a0b05ed66..f6d68377c 100644 --- a/pom.xml +++ b/pom.xml @@ -3,27 +3,35 @@ 4.0.0 org.qortal qortal - 1.2.2 + 3.1.1 jar - 0.15.5 + true + bf9fb80 + 0.15.10 1.64 ${maven.build.timestamp} - 1.3.4 + 1.3.8 3.6 1.8 + 2.6 + 1.21 + 3.12.0 + 1.9 1.2.2 28.1-jre - 2.5.0-fixed - 2.5.0 + 2.5.1 + 1.1 2.29.1 - 9.4.22.v20191022 - 2.12.1 + 9.4.29.v20200521 + 2.17.1 UTF-8 1.7.12 2.0.9 3.23.8 1.1.0 + 1.13.1 + 4.10 src/main/java @@ -199,6 +207,10 @@ org.qortal.api.model** + + org.qortal.api.model.** + + ${project.build.directory}/generated-sources/package-info @@ -313,6 +325,14 @@ + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + ${skipTests} + + @@ -375,6 +395,11 @@ project file:${project.basedir}/lib + + + jitpack.io + https://jitpack.io + @@ -397,35 +422,66 @@ hsqldb ${hsqldb.version} - - org.hsqldb - sqltool - ${hsqldb-sqltool.version} - test - org.ciyam AT ${ciyam-at.version} + + + com.dosse + WaifUPnP + ${upnp.version} + org.bitcoinj bitcoinj-core ${bitcoinj.version} + + + com.github.jjos2372 + altcoinj + ${altcoinj.version} + com.googlecode.json-simple json-simple 1.1.1 + + org.json + json + 20210307 + org.apache.commons commons-text ${commons-text.version} + + commons-io + commons-io + ${commons-io.version} + + + org.apache.commons + commons-compress + ${commons-compress.version} + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + org.tukaani + xz + ${xz.version} + io.druid @@ -626,5 +682,15 @@ bctls-jdk15on ${bouncycastle.version} + + org.jsoup + jsoup + ${jsoup.version} + + + io.github.java-diff-utils + java-diff-utils + ${java-diff-utils.version} + diff --git a/run.sh b/run.sh deleted file mode 100755 index 474c71391..000000000 --- a/run.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh - -# There's no need to run as root, so don't allow it, for security reasons -if [ "$USER" = "root" ]; then - echo "Please su to non-root user before running" - exit -fi - -# Validate Java is installed and the minimum version is available -MIN_JAVA_VER='11' - -if command -v java > /dev/null 2>&1; then - version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}') - version=$(echo $version | cut -d'.' -f1,2) - if [ `echo "${version}>=${MIN_JAVA_VER}" | bc` -eq 1 ]; then - echo 'Passed Java version check' - else - echo 'Please upgrade your Java to version 11 or greater' - exit 1 - fi -else - echo 'Java is not available, please install Java 11 or greater' - exit 1 -fi - -# No qortal.jar but we have a Maven built one? -# Be helpful and copy across to correct location -if [ ! -e qortal.jar -a -f target/qortal*.jar ]; then - echo "Copying Maven-built Qortal JAR to correct pathname" - cp target/qortal*.jar qortal.jar -fi - -# Limits Java JVM stack size and maximum heap usage. -# Comment out for bigger systems, e.g. non-routers -# or when API documentation is enabled -# JVM_MEMORY_ARGS="-Xss256k -Xmx128m" - -# Although java.net.preferIPv4Stack is supposed to be false -# by default in Java 11, on some platforms (e.g. FreeBSD 12), -# it is overriden to be true by default. Hence we explicitly -# set it to true to obtain desired behaviour. -nohup nice -n 20 java \ - -Djava.net.preferIPv4Stack=false \ - -XX:NativeMemoryTracking=summary \ - ${JVM_MEMORY_ARGS} \ - -jar qortal.jar \ - 1>run.log 2>&1 & - -# Save backgrounded process's PID -echo $! > run.pid -echo qortal running as pid $! diff --git a/run.sh b/run.sh new file mode 120000 index 000000000..ebcc7e664 --- /dev/null +++ b/run.sh @@ -0,0 +1 @@ +start.sh \ No newline at end of file diff --git a/src/main/java/org/hsqldb/jdbc/HSQLDBPool.java b/src/main/java/org/hsqldb/jdbc/HSQLDBPool.java index b7cdf653b..0bf9d2ef8 100644 --- a/src/main/java/org/hsqldb/jdbc/HSQLDBPool.java +++ b/src/main/java/org/hsqldb/jdbc/HSQLDBPool.java @@ -21,18 +21,28 @@ public HSQLDBPool(int poolSize) { public Connection tryConnection() throws SQLException { for (int i = 0; i < states.length(); i++) { if (states.compareAndSet(i, RefState.available, RefState.allocated)) { - return connections[i].getConnection(); + JDBCPooledConnection pooledConnection = connections[i]; + + if (pooledConnection == null) + // Probably shutdown situation + return null; + + return pooledConnection.getConnection(); } if (states.compareAndSet(i, RefState.empty, RefState.allocated)) { try { - JDBCPooledConnection connection = (JDBCPooledConnection) source.getPooledConnection(); + JDBCPooledConnection pooledConnection = (JDBCPooledConnection) source.getPooledConnection(); + + if (pooledConnection == null) + // Probably shutdown situation + return null; - connection.addConnectionEventListener(this); - connection.addStatementEventListener(this); - connections[i] = connection; + pooledConnection.addConnectionEventListener(this); + pooledConnection.addStatementEventListener(this); + connections[i] = pooledConnection; - return connections[i].getConnection(); + return pooledConnection.getConnection(); } catch (SQLException e) { states.set(i, RefState.empty); } diff --git a/src/main/java/org/qortal/ApplyUpdate.java b/src/main/java/org/qortal/ApplyUpdate.java index 0a432c4ea..901711919 100644 --- a/src/main/java/org/qortal/ApplyUpdate.java +++ b/src/main/java/org/qortal/ApplyUpdate.java @@ -7,14 +7,13 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.security.Security; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.qortal.api.ApiKey; import org.qortal.api.ApiRequest; import org.qortal.controller.AutoUpdate; import org.qortal.settings.Settings; @@ -35,6 +34,8 @@ public class ApplyUpdate { private static final String JAR_FILENAME = AutoUpdate.JAR_FILENAME; private static final String NEW_JAR_FILENAME = AutoUpdate.NEW_JAR_FILENAME; private static final String WINDOWS_EXE_LAUNCHER = "qortal.exe"; + private static final String JAVA_TOOL_OPTIONS_NAME = "JAVA_TOOL_OPTIONS"; + private static final String JAVA_TOOL_OPTIONS_VALUE = "-XX:MaxRAMFraction=4"; private static final long CHECK_INTERVAL = 10 * 1000L; // ms private static final int MAX_ATTEMPTS = 12; @@ -65,17 +66,45 @@ public static void main(String[] args) { } private static boolean shutdownNode() { - String BASE_URI = "http://localhost:" + Settings.getInstance().getApiPort() + "/"; - LOGGER.info(String.format("Shutting down node using API via %s", BASE_URI)); + String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/"; + LOGGER.info(() -> String.format("Shutting down node using API via %s", baseUri)); + // The /admin/stop endpoint requires an API key, which may or may not be already generated + boolean apiKeyNewlyGenerated = false; + ApiKey apiKey = null; + try { + apiKey = new ApiKey(); + if (!apiKey.generated()) { + apiKey.generate(); + apiKeyNewlyGenerated = true; + LOGGER.info("Generated API key"); + } + } catch (IOException e) { + LOGGER.info("Error loading API key: {}", e.getMessage()); + } + + // Create GET params + Map params = new HashMap<>(); + if (apiKey != null) { + params.put("apiKey", apiKey.toString()); + } + + // Attempt to stop the node int attempt; for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) { - LOGGER.info(String.format("Attempt #%d out of %d to shutdown node", attempt + 1, MAX_ATTEMPTS)); - String response = ApiRequest.perform(BASE_URI + "admin/stop", null); - if (response == null) - break; + final int attemptForLogging = attempt; + LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS)); + String response = ApiRequest.perform(baseUri + "admin/stop", params); + if (response == null) { + // No response - consider node shut down + if (apiKeyNewlyGenerated) { + // API key was newly generated for this auto update, so we need to remove it + ApplyUpdate.removeGeneratedApiKey(); + } + return true; + } - LOGGER.info(String.format("Response from API: %s", response)); + LOGGER.info(() -> String.format("Response from API: %s", response)); try { Thread.sleep(CHECK_INTERVAL); @@ -85,6 +114,11 @@ private static boolean shutdownNode() { } } + if (apiKeyNewlyGenerated) { + // API key was newly generated for this auto update, so we need to remove it + ApplyUpdate.removeGeneratedApiKey(); + } + if (attempt == MAX_ATTEMPTS) { LOGGER.error("Failed to shutdown node - giving up"); return false; @@ -93,25 +127,39 @@ private static boolean shutdownNode() { return true; } + private static void removeGeneratedApiKey() { + try { + LOGGER.info("Removing newly generated API key..."); + + // Delete the API key since it was only generated for this auto update + ApiKey apiKey = new ApiKey(); + apiKey.delete(); + + } catch (IOException e) { + LOGGER.info("Error loading or deleting API key: {}", e.getMessage()); + } + } + private static void replaceJar() { // Assuming current working directory contains the JAR files Path realJar = Paths.get(JAR_FILENAME); Path newJar = Paths.get(NEW_JAR_FILENAME); if (!Files.exists(newJar)) { - LOGGER.warn(String.format("Replacement JAR '%s' not found?", newJar)); + LOGGER.warn(() -> String.format("Replacement JAR '%s' not found?", newJar)); return; } int attempt; for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) { - LOGGER.info(String.format("Attempt #%d out of %d to replace JAR", attempt + 1, MAX_ATTEMPTS)); + final int attemptForLogging = attempt; + LOGGER.info(() -> String.format("Attempt #%d out of %d to replace JAR", attemptForLogging + 1, MAX_ATTEMPTS)); try { Files.copy(newJar, realJar, StandardCopyOption.REPLACE_EXISTING); break; } catch (IOException e) { - LOGGER.info(String.format("Unable to replace JAR: %s", e.getMessage())); + LOGGER.info(() -> String.format("Unable to replace JAR: %s", e.getMessage())); // Try again } @@ -119,6 +167,7 @@ private static void replaceJar() { try { Thread.sleep(CHECK_INTERVAL); } catch (InterruptedException e) { + LOGGER.warn("Ignoring interrupt..."); // Doggedly retry } } @@ -129,13 +178,13 @@ private static void replaceJar() { private static void restartNode(String[] args) { String javaHome = System.getProperty("java.home"); - LOGGER.info(String.format("Java home: %s", javaHome)); + LOGGER.info(() -> String.format("Java home: %s", javaHome)); Path javaBinary = Paths.get(javaHome, "bin", "java"); - LOGGER.info(String.format("Java binary: %s", javaBinary)); + LOGGER.info(() -> String.format("Java binary: %s", javaBinary)); Path exeLauncher = Paths.get(WINDOWS_EXE_LAUNCHER); - LOGGER.info(String.format("Windows EXE launcher: %s", exeLauncher)); + LOGGER.info(() -> String.format("Windows EXE launcher: %s", exeLauncher)); List javaCmd; if (Files.exists(exeLauncher)) { @@ -156,9 +205,16 @@ private static void restartNode(String[] args) { } try { - LOGGER.info(String.format("Restarting node with: %s", String.join(" ", javaCmd))); + LOGGER.info(() -> String.format("Restarting node with: %s", String.join(" ", javaCmd))); + + ProcessBuilder processBuilder = new ProcessBuilder(javaCmd); + + if (Files.exists(exeLauncher)) { + LOGGER.info(() -> String.format("Setting env %s to %s", JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE)); + processBuilder.environment().put(JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE); + } - new ProcessBuilder(javaCmd).start(); + processBuilder.start(); } catch (IOException e) { LOGGER.error(String.format("Failed to restart node (BAD): %s", e.getMessage())); } diff --git a/src/main/java/org/qortal/RepositoryMaintenance.java b/src/main/java/org/qortal/RepositoryMaintenance.java new file mode 100644 index 000000000..b085822b2 --- /dev/null +++ b/src/main/java/org/qortal/RepositoryMaintenance.java @@ -0,0 +1,76 @@ +package org.qortal; + +import java.security.Security; +import java.util.concurrent.TimeoutException; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.qortal.controller.Controller; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryFactory; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; +import org.qortal.settings.Settings; + +public class RepositoryMaintenance { + + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + } + + private static final Logger LOGGER = LogManager.getLogger(RepositoryMaintenance.class); + + public static void main(String[] args) { + LOGGER.info("Repository maintenance starting up..."); + + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + + // Load/check settings, which potentially sets up blockchain config, etc. + try { + if (args.length > 0) + Settings.fileInstance(args[0]); + else + Settings.getInstance(); + } catch (Throwable t) { + LOGGER.error("Settings file error: " + t.getMessage()); + System.exit(2); + } + + LOGGER.info("Opening repository"); + try { + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); + } catch (DataException e) { + // If exception has no cause then repository is in use by some other process. + if (e.getCause() == null) { + LOGGER.info("Repository in use by another process?"); + } else { + LOGGER.error("Unable to start repository", e); + } + + System.exit(1); + } + + LOGGER.info("Starting repository periodic maintenance. This can take a while..."); + try (final Repository repository = RepositoryManager.getRepository()) { + repository.performPeriodicMaintenance(null); + + LOGGER.info("Repository periodic maintenance completed"); + } catch (DataException | TimeoutException e) { + LOGGER.error("Repository periodic maintenance failed", e); + } + + try { + LOGGER.info("Shutting down repository"); + RepositoryManager.closeRepositoryFactory(); + } catch (DataException e) { + LOGGER.error("Error occurred while shutting down repository", e); + } + } + +} diff --git a/src/main/java/org/qortal/account/Account.java b/src/main/java/org/qortal/account/Account.java index 417dde6df..aeff78101 100644 --- a/src/main/java/org/qortal/account/Account.java +++ b/src/main/java/org/qortal/account/Account.java @@ -272,7 +272,7 @@ public int getEffectiveMintingLevel() throws DataException { /** * Returns 'effective' minting level, or zero if reward-share does not exist. *

- * For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config. + * this is being used on src/main/java/org/qortal/api/resource/AddressesResource.java to fulfil the online accounts api call * * @param repository * @param rewardSharePublicKey @@ -288,5 +288,26 @@ public static int getRewardShareEffectiveMintingLevel(Repository repository, byt Account rewardShareMinter = new Account(repository, rewardShareData.getMinter()); return rewardShareMinter.getEffectiveMintingLevel(); } + /** + * Returns 'effective' minting level, with a fix for the zero level. + *

+ * For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config. + * + * @param repository + * @param rewardSharePublicKey + * @return 0+ + * @throws DataException + */ + public static int getRewardShareEffectiveMintingLevelIncludingLevelZero(Repository repository, byte[] rewardSharePublicKey) throws DataException { + // Find actual minter and get their effective minting level + RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey); + if (rewardShareData == null) + return 0; + else if(!rewardShareData.getMinter().equals(rewardShareData.getRecipient()))//the minter is different than the recipient this means sponsorship + return 0; + + Account rewardShareMinter = new Account(repository, rewardShareData.getMinter()); + return rewardShareMinter.getEffectiveMintingLevel(); + } } diff --git a/src/main/java/org/qortal/api/ApiError.java b/src/main/java/org/qortal/api/ApiError.java index f39ff7a00..659104e70 100644 --- a/src/main/java/org/qortal/api/ApiError.java +++ b/src/main/java/org/qortal/api/ApiError.java @@ -15,7 +15,7 @@ public enum ApiError { // COMMON // UNKNOWN(0, 500), JSON(1, 400), - // NO_BALANCE(2, 422), + INSUFFICIENT_BALANCE(2, 402), // NOT_YET_RELEASED(3, 422), UNAUTHORIZED(4, 403), REPOSITORY_ISSUE(5, 500), @@ -126,10 +126,17 @@ public enum ApiError { // Groups GROUP_UNKNOWN(1101, 404), - // Bitcoin - BTC_NETWORK_ISSUE(1201, 500), - BTC_BALANCE_ISSUE(1202, 422), - BTC_TOO_SOON(1203, 422); + // Foreign blockchain + FOREIGN_BLOCKCHAIN_NETWORK_ISSUE(1201, 500), + FOREIGN_BLOCKCHAIN_BALANCE_ISSUE(1202, 402), + FOREIGN_BLOCKCHAIN_TOO_SOON(1203, 408), + + // Trade portal + ORDER_SIZE_TOO_SMALL(1300, 402), + + // Data + FILE_NOT_FOUND(1401, 404), + NO_REPLY(1402, 404); private static final Map map = stream(ApiError.values()).collect(toMap(apiError -> apiError.code, apiError -> apiError)); @@ -157,4 +164,4 @@ public int getStatus() { return this.status; } -} \ No newline at end of file +} diff --git a/src/main/java/org/qortal/api/ApiErrorHandler.java b/src/main/java/org/qortal/api/ApiErrorHandler.java index 38aef3c12..a6a77eedc 100644 --- a/src/main/java/org/qortal/api/ApiErrorHandler.java +++ b/src/main/java/org/qortal/api/ApiErrorHandler.java @@ -3,6 +3,7 @@ import java.io.IOException; import javax.servlet.RequestDispatcher; +import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -17,7 +18,7 @@ public class ApiErrorHandler extends ErrorHandler { private static final Logger LOGGER = LogManager.getLogger(ApiErrorHandler.class); @Override - public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { if (Settings.getInstance().isApiLoggingEnabled()) { String requestURI = request.getRequestURI(); diff --git a/src/main/java/org/qortal/api/ApiExceptionFactory.java b/src/main/java/org/qortal/api/ApiExceptionFactory.java index e66c6e843..294cef834 100644 --- a/src/main/java/org/qortal/api/ApiExceptionFactory.java +++ b/src/main/java/org/qortal/api/ApiExceptionFactory.java @@ -16,4 +16,8 @@ public ApiException createException(HttpServletRequest request, ApiError apiErro return createException(request, apiError, null); } + public ApiException createCustomException(HttpServletRequest request, ApiError apiError, String message) { + return new ApiException(apiError.getStatus(), apiError.getCode(), message, null); + } + } diff --git a/src/main/java/org/qortal/api/ApiKey.java b/src/main/java/org/qortal/api/ApiKey.java new file mode 100644 index 000000000..3f7cfe35b --- /dev/null +++ b/src/main/java/org/qortal/api/ApiKey.java @@ -0,0 +1,107 @@ +package org.qortal.api; + +import org.qortal.settings.Settings; +import org.qortal.utils.Base58; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.SecureRandom; + +public class ApiKey { + + private String apiKey; + + public ApiKey() throws IOException { + this.load(); + } + + public void generate() throws IOException { + byte[] apiKey = new byte[16]; + new SecureRandom().nextBytes(apiKey); + this.apiKey = Base58.encode(apiKey); + + this.save(); + } + + + /* Filesystem */ + + private Path getFilePath() { + return Paths.get(Settings.getInstance().getApiKeyPath(), "apikey.txt"); + } + + private boolean load() throws IOException { + Path path = this.getFilePath(); + File apiKeyFile = new File(path.toString()); + if (!apiKeyFile.exists()) { + // Try settings - to allow legacy API keys to be supported + return this.loadLegacyApiKey(); + } + + try { + this.apiKey = new String(Files.readAllBytes(path)); + + } catch (IOException e) { + throw new IOException(String.format("Couldn't read contents from file %s", path.toString())); + } + + return true; + } + + private boolean loadLegacyApiKey() { + String legacyApiKey = Settings.getInstance().getApiKey(); + if (legacyApiKey != null && !legacyApiKey.isEmpty()) { + this.apiKey = Settings.getInstance().getApiKey(); + + try { + // Save it to the apikey file + this.save(); + } catch (IOException e) { + // Ignore failures as it will be reloaded from settings next time + } + return true; + } + return false; + } + + public void save() throws IOException { + if (this.apiKey == null || this.apiKey.isEmpty()) { + throw new IllegalStateException("Unable to save a blank API key"); + } + + Path filePath = this.getFilePath(); + + BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toString())); + writer.write(this.apiKey); + writer.close(); + } + + public void delete() throws IOException { + this.apiKey = null; + + Path filePath = this.getFilePath(); + if (Files.exists(filePath)) { + Files.delete(filePath); + } + } + + + public boolean generated() { + return (this.apiKey != null); + } + + public boolean exists() { + return this.getFilePath().toFile().exists(); + } + + @Override + public String toString() { + return this.apiKey; + } + +} diff --git a/src/main/java/org/qortal/api/ApiService.java b/src/main/java/org/qortal/api/ApiService.java index 52b03cac5..697543c71 100644 --- a/src/main/java/org/qortal/api/ApiService.java +++ b/src/main/java/org/qortal/api/ApiService.java @@ -14,13 +14,14 @@ import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; +import org.checkerframework.checker.units.qual.A; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.rewrite.handler.RedirectPatternRule; import org.eclipse.jetty.rewrite.handler.RewriteHandler; import org.eclipse.jetty.server.CustomRequestLog; +import org.eclipse.jetty.server.DetectorConnectionFactory; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; -import org.eclipse.jetty.server.OptionalSslConnectionFactory; import org.eclipse.jetty.server.RequestLog; import org.eclipse.jetty.server.RequestLogWriter; import org.eclipse.jetty.server.SecureRequestCustomizer; @@ -43,6 +44,9 @@ import org.qortal.api.websocket.AdminStatusWebSocket; import org.qortal.api.websocket.BlocksWebSocket; import org.qortal.api.websocket.ChatMessagesWebSocket; +import org.qortal.api.websocket.PresenceWebSocket; +import org.qortal.api.websocket.TradeBotWebSocket; +import org.qortal.api.websocket.TradeOffersWebSocket; import org.qortal.settings.Settings; public class ApiService { @@ -51,6 +55,7 @@ public class ApiService { private final ResourceConfig config; private Server server; + private ApiKey apiKey; private ApiService() { this.config = new ResourceConfig(); @@ -71,6 +76,15 @@ public Iterable> getResources() { return this.config.getClasses(); } + public void setApiKey(ApiKey apiKey) { + this.apiKey = apiKey; + } + + public ApiKey getApiKey() { + return this.apiKey; + } + + public void start() { try { // Create API server @@ -113,8 +127,7 @@ public void start() { SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()); ServerConnector portUnifiedConnector = new ServerConnector(this.server, - new OptionalSslConnectionFactory(sslConnectionFactory, HttpVersion.HTTP_1_1.asString()), - sslConnectionFactory, + new DetectorConnectionFactory(sslConnectionFactory), httpConnectionFactory); portUnifiedConnector.setHost(Settings.getInstance().getBindAddress()); portUnifiedConnector.setPort(Settings.getInstance().getApiPort()); @@ -197,6 +210,9 @@ public void start() { context.addServlet(BlocksWebSocket.class, "/websockets/blocks"); context.addServlet(ActiveChatsWebSocket.class, "/websockets/chat/active/*"); context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages"); + context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers"); + context.addServlet(TradeBotWebSocket.class, "/websockets/crosschain/tradebot"); + context.addServlet(PresenceWebSocket.class, "/websockets/presence"); // Start server this.server.start(); diff --git a/src/main/java/org/qortal/api/Base58TypeAdapter.java b/src/main/java/org/qortal/api/Base58TypeAdapter.java index 4b292a2a5..d75610317 100644 --- a/src/main/java/org/qortal/api/Base58TypeAdapter.java +++ b/src/main/java/org/qortal/api/Base58TypeAdapter.java @@ -2,7 +2,7 @@ import javax.xml.bind.annotation.adapters.XmlAdapter; -import org.bitcoinj.core.Base58; +import org.qortal.utils.Base58; public class Base58TypeAdapter extends XmlAdapter { diff --git a/src/main/java/org/qortal/api/DomainMapService.java b/src/main/java/org/qortal/api/DomainMapService.java new file mode 100644 index 000000000..ba0fa0678 --- /dev/null +++ b/src/main/java/org/qortal/api/DomainMapService.java @@ -0,0 +1,171 @@ +package org.qortal.api; + +import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.rewrite.handler.RewriteHandler; +import org.eclipse.jetty.rewrite.handler.RewritePatternRule; +import org.eclipse.jetty.server.*; +import org.eclipse.jetty.server.handler.ErrorHandler; +import org.eclipse.jetty.server.handler.InetAccessHandler; +import org.eclipse.jetty.servlet.FilterHolder; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.servlets.CrossOriginFilter; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.servlet.ServletContainer; +import org.qortal.api.resource.AnnotationPostProcessor; +import org.qortal.api.resource.ApiDefinition; +import org.qortal.settings.Settings; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.security.SecureRandom; + +public class DomainMapService { + + private static DomainMapService instance; + + private final ResourceConfig config; + private Server server; + + private DomainMapService() { + this.config = new ResourceConfig(); + this.config.packages("org.qortal.api.domainmap.resource"); + this.config.register(OpenApiResource.class); + this.config.register(ApiDefinition.class); + this.config.register(AnnotationPostProcessor.class); + } + + public static DomainMapService getInstance() { + if (instance == null) + instance = new DomainMapService(); + + return instance; + } + + public Iterable> getResources() { + return this.config.getClasses(); + } + + public void start() { + try { + // Create API server + + // SSL support if requested + String keystorePathname = Settings.getInstance().getSslKeystorePathname(); + String keystorePassword = Settings.getInstance().getSslKeystorePassword(); + + if (keystorePathname != null && keystorePassword != null) { + // SSL version + if (!Files.isReadable(Path.of(keystorePathname))) + throw new RuntimeException("Failed to start SSL API due to broken keystore"); + + // BouncyCastle-specific SSLContext build + SSLContext sslContext = SSLContext.getInstance("TLS", "BCJSSE"); + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE"); + + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC"); + + try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) { + keyStore.load(keystoreStream, keystorePassword.toCharArray()); + } + + keyManagerFactory.init(keyStore, keystorePassword.toCharArray()); + sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom()); + + SslContextFactory.Server sslContextFactory = new SslContextFactory.Server(); + sslContextFactory.setSslContext(sslContext); + + this.server = new Server(); + + HttpConfiguration httpConfig = new HttpConfiguration(); + httpConfig.setSecureScheme("https"); + httpConfig.setSecurePort(Settings.getInstance().getDomainMapPort()); + + SecureRequestCustomizer src = new SecureRequestCustomizer(); + httpConfig.addCustomizer(src); + + HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig); + SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()); + + ServerConnector portUnifiedConnector = new ServerConnector(this.server, + new DetectorConnectionFactory(sslConnectionFactory), + httpConnectionFactory); + portUnifiedConnector.setHost(Settings.getInstance().getBindAddress()); + portUnifiedConnector.setPort(Settings.getInstance().getDomainMapPort()); + + this.server.addConnector(portUnifiedConnector); + } else { + // Non-SSL + InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress()); + InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getDomainMapPort()); + this.server = new Server(endpoint); + } + + // Error handler + ErrorHandler errorHandler = new ApiErrorHandler(); + this.server.setErrorHandler(errorHandler); + + // Request logging + if (Settings.getInstance().isDomainMapLoggingEnabled()) { + RequestLogWriter logWriter = new RequestLogWriter("domainmap-requests.log"); + logWriter.setAppend(true); + logWriter.setTimeZone("UTC"); + RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT); + this.server.setRequestLog(requestLog); + } + + // Access handler (currently no whitelist is used) + InetAccessHandler accessHandler = new InetAccessHandler(); + this.server.setHandler(accessHandler); + + // URL rewriting + RewriteHandler rewriteHandler = new RewriteHandler(); + accessHandler.setHandler(rewriteHandler); + + // Context + ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + context.setContextPath("/"); + rewriteHandler.setHandler(context); + + // Cross-origin resource sharing + FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class); + corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*"); + corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE"); + corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false"); + context.addFilter(corsFilterHolder, "/*", null); + + // API servlet + ServletContainer container = new ServletContainer(this.config); + ServletHolder apiServlet = new ServletHolder(container); + apiServlet.setInitOrder(1); + context.addServlet(apiServlet, "/*"); + + // Start server + this.server.start(); + } catch (Exception e) { + // Failed to start + throw new RuntimeException("Failed to start API", e); + } + } + + public void stop() { + try { + // Stop server + this.server.stop(); + } catch (Exception e) { + // Failed to stop + } + + this.server = null; + } + +} diff --git a/src/main/java/org/qortal/api/GatewayService.java b/src/main/java/org/qortal/api/GatewayService.java new file mode 100644 index 000000000..030a0f2f5 --- /dev/null +++ b/src/main/java/org/qortal/api/GatewayService.java @@ -0,0 +1,170 @@ +package org.qortal.api; + +import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.rewrite.handler.RewriteHandler; +import org.eclipse.jetty.server.*; +import org.eclipse.jetty.server.handler.ErrorHandler; +import org.eclipse.jetty.server.handler.InetAccessHandler; +import org.eclipse.jetty.servlet.FilterHolder; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.servlets.CrossOriginFilter; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.servlet.ServletContainer; +import org.qortal.api.resource.AnnotationPostProcessor; +import org.qortal.api.resource.ApiDefinition; +import org.qortal.settings.Settings; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.security.SecureRandom; + +public class GatewayService { + + private static GatewayService instance; + + private final ResourceConfig config; + private Server server; + + private GatewayService() { + this.config = new ResourceConfig(); + this.config.packages("org.qortal.api.gateway.resource"); + this.config.register(OpenApiResource.class); + this.config.register(ApiDefinition.class); + this.config.register(AnnotationPostProcessor.class); + } + + public static GatewayService getInstance() { + if (instance == null) + instance = new GatewayService(); + + return instance; + } + + public Iterable> getResources() { + return this.config.getClasses(); + } + + public void start() { + try { + // Create API server + + // SSL support if requested + String keystorePathname = Settings.getInstance().getSslKeystorePathname(); + String keystorePassword = Settings.getInstance().getSslKeystorePassword(); + + if (keystorePathname != null && keystorePassword != null) { + // SSL version + if (!Files.isReadable(Path.of(keystorePathname))) + throw new RuntimeException("Failed to start SSL API due to broken keystore"); + + // BouncyCastle-specific SSLContext build + SSLContext sslContext = SSLContext.getInstance("TLS", "BCJSSE"); + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE"); + + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC"); + + try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) { + keyStore.load(keystoreStream, keystorePassword.toCharArray()); + } + + keyManagerFactory.init(keyStore, keystorePassword.toCharArray()); + sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom()); + + SslContextFactory.Server sslContextFactory = new SslContextFactory.Server(); + sslContextFactory.setSslContext(sslContext); + + this.server = new Server(); + + HttpConfiguration httpConfig = new HttpConfiguration(); + httpConfig.setSecureScheme("https"); + httpConfig.setSecurePort(Settings.getInstance().getGatewayPort()); + + SecureRequestCustomizer src = new SecureRequestCustomizer(); + httpConfig.addCustomizer(src); + + HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig); + SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()); + + ServerConnector portUnifiedConnector = new ServerConnector(this.server, + new DetectorConnectionFactory(sslConnectionFactory), + httpConnectionFactory); + portUnifiedConnector.setHost(Settings.getInstance().getBindAddress()); + portUnifiedConnector.setPort(Settings.getInstance().getGatewayPort()); + + this.server.addConnector(portUnifiedConnector); + } else { + // Non-SSL + InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress()); + InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getGatewayPort()); + this.server = new Server(endpoint); + } + + // Error handler + ErrorHandler errorHandler = new ApiErrorHandler(); + this.server.setErrorHandler(errorHandler); + + // Request logging + if (Settings.getInstance().isGatewayLoggingEnabled()) { + RequestLogWriter logWriter = new RequestLogWriter("gateway-requests.log"); + logWriter.setAppend(true); + logWriter.setTimeZone("UTC"); + RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT); + this.server.setRequestLog(requestLog); + } + + // Access handler (currently no whitelist is used) + InetAccessHandler accessHandler = new InetAccessHandler(); + this.server.setHandler(accessHandler); + + // URL rewriting + RewriteHandler rewriteHandler = new RewriteHandler(); + accessHandler.setHandler(rewriteHandler); + + // Context + ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + context.setContextPath("/"); + rewriteHandler.setHandler(context); + + // Cross-origin resource sharing + FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class); + corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*"); + corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE"); + corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false"); + context.addFilter(corsFilterHolder, "/*", null); + + // API servlet + ServletContainer container = new ServletContainer(this.config); + ServletHolder apiServlet = new ServletHolder(container); + apiServlet.setInitOrder(1); + context.addServlet(apiServlet, "/*"); + + // Start server + this.server.start(); + } catch (Exception e) { + // Failed to start + throw new RuntimeException("Failed to start API", e); + } + } + + public void stop() { + try { + // Stop server + this.server.stop(); + } catch (Exception e) { + // Failed to stop + } + + this.server = null; + } + +} diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java new file mode 100644 index 000000000..026d92102 --- /dev/null +++ b/src/main/java/org/qortal/api/HTMLParser.java @@ -0,0 +1,51 @@ +package org.qortal.api; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.select.Elements; + +public class HTMLParser { + + private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class); + + private String linkPrefix; + private byte[] data; + + public HTMLParser(String resourceId, String inPath, String prefix, boolean usePrefix, byte[] data) { + String inPathWithoutFilename = inPath.substring(0, inPath.lastIndexOf('/')); + this.linkPrefix = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : ""; + this.data = data; + } + + public void addAdditionalHeaderTags() { + String fileContents = new String(data); + Document document = Jsoup.parse(fileContents); + String baseUrl = this.linkPrefix + "/"; + Elements head = document.getElementsByTag("head"); + if (!head.isEmpty()) { + // Add base href tag + String baseElement = String.format("", baseUrl); + head.get(0).prepend(baseElement); + + // Add meta charset tag + String metaCharsetElement = ""; + head.get(0).prepend(metaCharsetElement); + + } + String html = document.html(); + this.data = html.getBytes(); + } + + public static boolean isHtmlFile(String path) { + if (path.endsWith(".html") || path.endsWith(".htm")) { + return true; + } + return false; + } + + public byte[] getData() { + return this.data; + } +} diff --git a/src/main/java/org/qortal/api/Security.java b/src/main/java/org/qortal/api/Security.java index 2449f781b..4aca2c49c 100644 --- a/src/main/java/org/qortal/api/Security.java +++ b/src/main/java/org/qortal/api/Security.java @@ -1,22 +1,111 @@ package org.qortal.api; +import org.qortal.arbitrary.ArbitraryDataResource; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.arbitrary.ArbitraryDataRenderManager; +import org.qortal.settings.Settings; + +import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import javax.servlet.http.HttpServletRequest; -public class Security { +public abstract class Security { + + public static final String API_KEY_HEADER = "X-API-KEY"; - // TODO: replace with proper authentication public static void checkApiCallAllowed(HttpServletRequest request) { - InetAddress remoteAddr; + // We may want to allow automatic authentication for local requests, if enabled in settings + boolean localAuthBypassEnabled = Settings.getInstance().isLocalAuthBypassEnabled(); + if (localAuthBypassEnabled) { + try { + InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr()); + if (remoteAddr.isLoopbackAddress()) { + // Request originates from loopback address, so allow it + return; + } + } catch (UnknownHostException e) { + // Ignore failure, and fallback to API key authentication + } + } + + // Retrieve the API key + ApiKey apiKey = Security.getApiKey(request); + if (!apiKey.generated()) { + // Not generated an API key yet, so disallow sensitive API calls + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "API key not generated"); + } + + // We require an API key to be passed + String passedApiKey = request.getHeader(API_KEY_HEADER); + if (passedApiKey == null) { + // Try query string - this is needed to avoid a CORS preflight. See: https://stackoverflow.com/a/43881141 + passedApiKey = request.getParameter("apiKey"); + } + if (passedApiKey == null) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Missing 'X-API-KEY' header"); + } + + // The API keys must match + if (!apiKey.toString().equals(passedApiKey)) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "API key invalid"); + } + } + + public static void disallowLoopbackRequests(HttpServletRequest request) { try { - remoteAddr = InetAddress.getByName(request.getRemoteAddr()); + InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr()); + if (remoteAddr.isLoopbackAddress()) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Local requests not allowed"); + } } catch (UnknownHostException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED); } + } - if (!remoteAddr.isLoopbackAddress()) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED); + public static void disallowLoopbackRequestsIfAuthBypassEnabled(HttpServletRequest request) { + if (Settings.getInstance().isLocalAuthBypassEnabled()) { + try { + InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr()); + if (remoteAddr.isLoopbackAddress()) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Local requests not allowed when localAuthBypassEnabled is enabled in settings"); + } + } catch (UnknownHostException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED); + } + } + } + + public static void requirePriorAuthorization(HttpServletRequest request, String resourceId, Service service, String identifier) { + ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, identifier); + if (!ArbitraryDataRenderManager.getInstance().isAuthorized(resource)) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Call /render/authorize first"); + } } + + public static void requirePriorAuthorizationOrApiKey(HttpServletRequest request, String resourceId, Service service, String identifier) { + try { + Security.checkApiCallAllowed(request); + + } catch (ApiException e) { + // API call wasn't allowed, but maybe it was pre-authorized + Security.requirePriorAuthorization(request, resourceId, service, identifier); + } + } + + public static ApiKey getApiKey(HttpServletRequest request) { + ApiKey apiKey = ApiService.getInstance().getApiKey(); + if (apiKey == null) { + try { + apiKey = new ApiKey(); + } catch (IOException e) { + // Couldn't load API key - so we need to treat it as not generated, and therefore unauthorized + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED); + } + ApiService.getInstance().setApiKey(apiKey); + } + return apiKey; + } + } diff --git a/src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java b/src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java new file mode 100644 index 000000000..cc21587d5 --- /dev/null +++ b/src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java @@ -0,0 +1,58 @@ +package org.qortal.api.domainmap.resource; + +import io.swagger.v3.oas.annotations.tags.Tag; +import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; +import org.qortal.arbitrary.ArbitraryDataRenderer; +import org.qortal.arbitrary.misc.Service; +import org.qortal.settings.Settings; + +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.core.Context; +import java.util.Map; + + +@Path("/") +@Tag(name = "Domain Map") +public class DomainMapResource { + + @Context HttpServletRequest request; + @Context HttpServletResponse response; + @Context ServletContext context; + + + @GET + public HttpServletResponse getIndexByDomainMap() { + return this.getDomainMap("/"); + } + + @GET + @Path("{path:.*}") + public HttpServletResponse getPathByDomainMap(@PathParam("path") String inPath) { + return this.getDomainMap(inPath); + } + + private HttpServletResponse getDomainMap(String inPath) { + Map domainMap = Settings.getInstance().getSimpleDomainMap(); + if (domainMap != null && domainMap.containsKey(request.getServerName())) { + // Build synchronously, so that we don't need to make the summary API endpoints available over + // the domain map server. This means that there will be no loading screen, but this is potentially + // preferred in this situation anyway (e.g. to avoid confusing search engine robots). + return this.get(domainMap.get(request.getServerName()), ResourceIdType.NAME, Service.WEBSITE, inPath, null, "", false, false); + } + return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found"); + } + + private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String inPath, + String secret58, String prefix, boolean usePrefix, boolean async) { + + ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, inPath, + secret58, prefix, usePrefix, async, request, response, context); + return renderer.render(); + } + +} diff --git a/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java b/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java new file mode 100644 index 000000000..a73de1fb4 --- /dev/null +++ b/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java @@ -0,0 +1,126 @@ +package org.qortal.api.gateway.resource; + +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.qortal.api.Security; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; +import org.qortal.arbitrary.ArbitraryDataReader; +import org.qortal.arbitrary.ArbitraryDataRenderer; +import org.qortal.arbitrary.ArbitraryDataResource; +import org.qortal.arbitrary.misc.Service; +import org.qortal.data.arbitrary.ArbitraryResourceStatus; + +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; + + +@Path("/") +@Tag(name = "Gateway") +public class GatewayResource { + + @Context HttpServletRequest request; + @Context HttpServletResponse response; + @Context ServletContext context; + + /** + * We need to allow resource status checking (and building) via the gateway, as the node's API port + * may not be forwarded and will almost certainly not be authenticated. Since gateways allow for + * all resources to be loaded except those that are blocked, there is no need for authentication. + */ + @GET + @Path("/arbitrary/resource/status/{service}/{name}") + public ArbitraryResourceStatus getDefaultResourceStatus(@PathParam("service") Service service, + @PathParam("name") String name, + @QueryParam("build") Boolean build) { + + return this.getStatus(service, name, null, build); + } + + @GET + @Path("/arbitrary/resource/status/{service}/{name}/{identifier}") + public ArbitraryResourceStatus getResourceStatus(@PathParam("service") Service service, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @QueryParam("build") Boolean build) { + + return this.getStatus(service, name, identifier, build); + } + + private ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build) { + + // If "build=true" has been specified in the query string, build the resource before returning its status + if (build != null && build == true) { + ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null); + try { + if (!reader.isBuilding()) { + reader.loadSynchronously(false); + } + } catch (Exception e) { + // No need to handle exception, as it will be reflected in the status + } + } + + ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); + return resource.getStatus(false); + } + + + @GET + public HttpServletResponse getRoot() { + return ArbitraryDataRenderer.getResponse(response, 200, ""); + } + + + @GET + @Path("{name}/{path:.*}") + @SecurityRequirement(name = "apiKey") + public HttpServletResponse getPathByName(@PathParam("name") String name, + @PathParam("path") String inPath) { + // Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data + Security.disallowLoopbackRequests(request); + return this.get(name, ResourceIdType.NAME, Service.WEBSITE, inPath, null, "", true, true); + } + + @GET + @Path("{name}") + @SecurityRequirement(name = "apiKey") + public HttpServletResponse getIndexByName(@PathParam("name") String name) { + // Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data + Security.disallowLoopbackRequests(request); + return this.get(name, ResourceIdType.NAME, Service.WEBSITE, "/", null, "", true, true); + } + + + // Optional /site alternative for backwards support + + @GET + @Path("/site/{name}/{path:.*}") + public HttpServletResponse getSitePathByName(@PathParam("name") String name, + @PathParam("path") String inPath) { + // Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data + Security.disallowLoopbackRequests(request); + return this.get(name, ResourceIdType.NAME, Service.WEBSITE, inPath, null, "/site", true, true); + } + + @GET + @Path("/site/{name}") + public HttpServletResponse getSiteIndexByName(@PathParam("name") String name) { + // Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data + Security.disallowLoopbackRequests(request); + return this.get(name, ResourceIdType.NAME, Service.WEBSITE, "/", null, "/site", true, true); + } + + + private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String inPath, + String secret58, String prefix, boolean usePrefix, boolean async) { + + ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, inPath, + secret58, prefix, usePrefix, async, request, response, context); + return renderer.render(); + } + +} diff --git a/src/main/java/org/qortal/api/model/ActivitySummary.java b/src/main/java/org/qortal/api/model/ActivitySummary.java index 27b5ed8dc..e8e9a3aaa 100644 --- a/src/main/java/org/qortal/api/model/ActivitySummary.java +++ b/src/main/java/org/qortal/api/model/ActivitySummary.java @@ -1,5 +1,6 @@ package org.qortal.api.model; +import java.util.Collections; import java.util.EnumMap; import java.util.Map; @@ -13,17 +14,61 @@ @XmlAccessorType(XmlAccessType.FIELD) public class ActivitySummary { - public int blockCount; - public int transactionCount; - public int assetsIssued; - public int namesRegistered; + private int blockCount; + private int assetsIssued; + private int namesRegistered; // Assuming TransactionType values are contiguous so 'length' equals count @XmlJavaTypeAdapter(TransactionCountMapXmlAdapter.class) - public Map transactionCountByType = new EnumMap<>(TransactionType.class); + private Map transactionCountByType = new EnumMap<>(TransactionType.class); + private int totalTransactionCount = 0; public ActivitySummary() { // Needed for JAXB } + public int getBlockCount() { + return this.blockCount; + } + + public void setBlockCount(int blockCount) { + this.blockCount = blockCount; + } + + public int getTotalTransactionCount() { + return this.totalTransactionCount; + } + + public int getAssetsIssued() { + return this.assetsIssued; + } + + public void setAssetsIssued(int assetsIssued) { + this.assetsIssued = assetsIssued; + } + + public int getNamesRegistered() { + return this.namesRegistered; + } + + public void setNamesRegistered(int namesRegistered) { + this.namesRegistered = namesRegistered; + } + + public Map getTransactionCountByType() { + return Collections.unmodifiableMap(this.transactionCountByType); + } + + public void setTransactionCountByType(TransactionType transactionType, int transactionCount) { + this.transactionCountByType.put(transactionType, transactionCount); + + this.totalTransactionCount = this.transactionCountByType.values().stream().mapToInt(Integer::intValue).sum(); + } + + public void setTransactionCountByType(Map transactionCountByType) { + this.transactionCountByType = new EnumMap<>(transactionCountByType); + + this.totalTransactionCount = this.transactionCountByType.values().stream().mapToInt(Integer::intValue).sum(); + } + } diff --git a/src/main/java/org/qortal/api/model/BlockMintingInfo.java b/src/main/java/org/qortal/api/model/BlockMintingInfo.java new file mode 100644 index 000000000..f84e179e1 --- /dev/null +++ b/src/main/java/org/qortal/api/model/BlockMintingInfo.java @@ -0,0 +1,23 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.math.BigDecimal; +import java.math.BigInteger; + +@XmlAccessorType(XmlAccessType.FIELD) +public class BlockMintingInfo { + + public byte[] minterPublicKey; + public int minterLevel; + public int onlineAccountsCount; + public BigDecimal maxDistance; + public BigInteger keyDistance; + public double keyDistanceRatio; + public long timestamp; + public long timeDelta; + + public BlockMintingInfo() { + } + +} diff --git a/src/main/java/org/qortal/api/model/ConnectedPeer.java b/src/main/java/org/qortal/api/model/ConnectedPeer.java index 3209ee6ae..21bfc1f98 100644 --- a/src/main/java/org/qortal/api/model/ConnectedPeer.java +++ b/src/main/java/org/qortal/api/model/ConnectedPeer.java @@ -1,61 +1,74 @@ package org.qortal.api.model; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - +import io.swagger.v3.oas.annotations.media.Schema; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; import org.qortal.network.Handshake; import org.qortal.network.Peer; -import io.swagger.v3.oas.annotations.media.Schema; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.UUID; +import java.util.concurrent.TimeUnit; @XmlAccessorType(XmlAccessType.FIELD) public class ConnectedPeer { - public enum Direction { - INBOUND, - OUTBOUND; - } - public Direction direction; - public Handshake handshakeStatus; - public Long lastPing; - public Long connectedWhen; - public Long peersConnectedWhen; - - public String address; - public String version; - - public String nodeId; - - public Integer lastHeight; - @Schema(example = "base58") - public byte[] lastBlockSignature; - public Long lastBlockTimestamp; - - protected ConnectedPeer() { - } - - public ConnectedPeer(Peer peer) { - this.direction = peer.isOutbound() ? Direction.OUTBOUND : Direction.INBOUND; - this.handshakeStatus = peer.getHandshakeStatus(); - this.lastPing = peer.getLastPing(); - - PeerData peerData = peer.getPeerData(); - this.connectedWhen = peer.getConnectionTimestamp(); - this.peersConnectedWhen = peer.getPeersConnectionTimestamp(); - - this.address = peerData.getAddress().toString(); - - this.version = peer.getPeersVersionString(); - this.nodeId = peer.getPeersNodeId(); - - PeerChainTipData peerChainTipData = peer.getChainTipData(); - if (peerChainTipData != null) { - this.lastHeight = peerChainTipData.getLastHeight(); - this.lastBlockSignature = peerChainTipData.getLastBlockSignature(); - this.lastBlockTimestamp = peerChainTipData.getLastBlockTimestamp(); - } - } + public enum Direction { + INBOUND, + OUTBOUND; + } + + public Direction direction; + public Handshake handshakeStatus; + public Long lastPing; + public Long connectedWhen; + public Long peersConnectedWhen; + + public String address; + public String version; + + public String nodeId; + + public Integer lastHeight; + @Schema(example = "base58") + public byte[] lastBlockSignature; + public Long lastBlockTimestamp; + public UUID connectionId; + public String age; + + protected ConnectedPeer() { + } + + public ConnectedPeer(Peer peer) { + this.direction = peer.isOutbound() ? Direction.OUTBOUND : Direction.INBOUND; + this.handshakeStatus = peer.getHandshakeStatus(); + this.lastPing = peer.getLastPing(); + + PeerData peerData = peer.getPeerData(); + this.connectedWhen = peer.getConnectionTimestamp(); + this.peersConnectedWhen = peer.getPeersConnectionTimestamp(); + + this.address = peerData.getAddress().toString(); + + this.version = peer.getPeersVersionString(); + this.nodeId = peer.getPeersNodeId(); + this.connectionId = peer.getPeerConnectionId(); + if (peer.getConnectionEstablishedTime() > 0) { + long age = (System.currentTimeMillis() - peer.getConnectionEstablishedTime()); + long minutes = TimeUnit.MILLISECONDS.toMinutes(age); + long seconds = TimeUnit.MILLISECONDS.toSeconds(age) - TimeUnit.MINUTES.toSeconds(minutes); + this.age = String.format("%dm %ds", minutes, seconds); + } else { + this.age = "connecting..."; + } + + PeerChainTipData peerChainTipData = peer.getChainTipData(); + if (peerChainTipData != null) { + this.lastHeight = peerChainTipData.getLastHeight(); + this.lastBlockSignature = peerChainTipData.getLastBlockSignature(); + this.lastBlockTimestamp = peerChainTipData.getLastBlockTimestamp(); + } + } } diff --git a/src/main/java/org/qortal/api/model/CrossChainBitcoinP2SHStatus.java b/src/main/java/org/qortal/api/model/CrossChainBitcoinP2SHStatus.java deleted file mode 100644 index ff986e869..000000000 --- a/src/main/java/org/qortal/api/model/CrossChainBitcoinP2SHStatus.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.qortal.api.model; - -import java.math.BigDecimal; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainBitcoinP2SHStatus { - - @Schema(description = "Bitcoin P2SH address", example = "3CdH27kTpV8dcFHVRYjQ8EEV5FJg9X8pSJ (mainnet), 2fMiRRXVsxhZeyfum9ifybZvaMHbQTmwdZw (testnet)") - public String bitcoinP2shAddress; - - @Schema(description = "Bitcoin P2SH balance") - public BigDecimal bitcoinP2shBalance; - - @Schema(description = "Can P2SH redeem yet?") - public boolean canRedeem; - - @Schema(description = "Can P2SH refund yet?") - public boolean canRefund; - - @Schema(description = "Secret extracted by P2SH redeemer") - public byte[] secret; - - public CrossChainBitcoinP2SHStatus() { - } - -} diff --git a/src/main/java/org/qortal/api/model/CrossChainBitcoinRedeemRequest.java b/src/main/java/org/qortal/api/model/CrossChainBitcoinRedeemRequest.java index 5e95e36ca..074fd24db 100644 --- a/src/main/java/org/qortal/api/model/CrossChainBitcoinRedeemRequest.java +++ b/src/main/java/org/qortal/api/model/CrossChainBitcoinRedeemRequest.java @@ -25,6 +25,9 @@ public class CrossChainBitcoinRedeemRequest { @Schema(description = "32-byte secret", example = "6gVbAXCVzJXAWwtAVGAfgAkkXpeXvPUwSciPmCfSfXJG") public byte[] secret; + @Schema(description = "Bitcoin HASH160(public key) for receiving funds, or omit to derive from private key", example = "u17kBVKkKSp12oUzaxFwNnq1JZf") + public byte[] receivingAccountInfo; + public CrossChainBitcoinRedeemRequest() { } diff --git a/src/main/java/org/qortal/api/model/CrossChainBitcoinRefundRequest.java b/src/main/java/org/qortal/api/model/CrossChainBitcoinRefundRequest.java index 541cb8c12..f24853890 100644 --- a/src/main/java/org/qortal/api/model/CrossChainBitcoinRefundRequest.java +++ b/src/main/java/org/qortal/api/model/CrossChainBitcoinRefundRequest.java @@ -22,6 +22,9 @@ public class CrossChainBitcoinRefundRequest { @Schema(description = "Bitcoin miner fee", example = "0.00001000") public BigDecimal bitcoinMinerFee; + @Schema(description = "Bitcoin HASH160(public key) for receiving funds, or omit to derive from private key", example = "u17kBVKkKSp12oUzaxFwNnq1JZf") + public byte[] receivingAccountInfo; + public CrossChainBitcoinRefundRequest() { } diff --git a/src/main/java/org/qortal/api/model/CrossChainBitcoinyHTLCStatus.java b/src/main/java/org/qortal/api/model/CrossChainBitcoinyHTLCStatus.java new file mode 100644 index 000000000..2772eae12 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainBitcoinyHTLCStatus.java @@ -0,0 +1,31 @@ +package org.qortal.api.model; + +import java.math.BigDecimal; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainBitcoinyHTLCStatus { + + @Schema(description = "P2SH address", example = "3CdH27kTpV8dcFHVRYjQ8EEV5FJg9X8pSJ (mainnet), 2fMiRRXVsxhZeyfum9ifybZvaMHbQTmwdZw (testnet)") + public String bitcoinP2shAddress; + + @Schema(description = "P2SH balance") + public BigDecimal bitcoinP2shBalance; + + @Schema(description = "Can HTLC redeem yet?") + public boolean canRedeem; + + @Schema(description = "Can HTLC refund yet?") + public boolean canRefund; + + @Schema(description = "Secret used by HTLC redeemer") + public byte[] secret; + + public CrossChainBitcoinyHTLCStatus() { + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainBuildRequest.java b/src/main/java/org/qortal/api/model/CrossChainBuildRequest.java index c4fa097a2..e8d38703f 100644 --- a/src/main/java/org/qortal/api/model/CrossChainBuildRequest.java +++ b/src/main/java/org/qortal/api/model/CrossChainBuildRequest.java @@ -12,20 +12,19 @@ public class CrossChainBuildRequest { @Schema(description = "AT creator's public key", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") public byte[] creatorPublicKey; - @Schema(description = "Initial QORT amount paid when trade agreed", example = "0.00100000") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long initialQortAmount; - @Schema(description = "Final QORT amount paid out on successful trade", example = "80.40200000") @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long finalQortAmount; + public long qortAmount; @Schema(description = "QORT amount funding AT, including covering AT execution fees", example = "123.45670000") @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) public long fundingQortAmount; + @Schema(description = "HASH160 of creator's Bitcoin public key", example = "2daMveGc5pdjRyFacbxBzMksCbyC") + public byte[] bitcoinPublicKeyHash; + @Schema(description = "HASH160 of secret", example = "43vnftqkjxrhb5kJdkU1ZFQLEnWV") - public byte[] secretHash; + public byte[] hashOfSecretB; @Schema(description = "Bitcoin P2SH BTC balance for release of secret", example = "0.00864200") @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) diff --git a/src/main/java/org/qortal/api/model/CrossChainCancelRequest.java b/src/main/java/org/qortal/api/model/CrossChainCancelRequest.java index e1f57a7e1..25a189520 100644 --- a/src/main/java/org/qortal/api/model/CrossChainCancelRequest.java +++ b/src/main/java/org/qortal/api/model/CrossChainCancelRequest.java @@ -8,10 +8,10 @@ @XmlAccessorType(XmlAccessType.FIELD) public class CrossChainCancelRequest { - @Schema(description = "AT creator's public key", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") + @Schema(description = "AT creator's public key", example = "K6wuddsBV3HzRrXFFezE7P5MoRXp5m3mEDokRDGZB6ry") public byte[] creatorPublicKey; - @Schema(description = "Qortal AT address") + @Schema(description = "Qortal trade AT address") public String atAddress; public CrossChainCancelRequest() { diff --git a/src/main/java/org/qortal/api/model/CrossChainDualSecretRequest.java b/src/main/java/org/qortal/api/model/CrossChainDualSecretRequest.java new file mode 100644 index 000000000..b6705d5d3 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainDualSecretRequest.java @@ -0,0 +1,29 @@ +package org.qortal.api.model; + +import io.swagger.v3.oas.annotations.media.Schema; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainDualSecretRequest { + + @Schema(description = "Public key to match AT's trade 'partner'", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") + public byte[] partnerPublicKey; + + @Schema(description = "Qortal AT address") + public String atAddress; + + @Schema(description = "secret-A (32 bytes)", example = "FHMzten4he9jZ4HGb4297Utj6F5g2w7serjq2EnAg2s1") + public byte[] secretA; + + @Schema(description = "secret-B (32 bytes)", example = "EN2Bgx3BcEMtxFCewmCVSMkfZjVKYhx3KEXC5A21KBGx") + public byte[] secretB; + + @Schema(description = "Qortal address for receiving QORT from AT") + public String receivingAddress; + + public CrossChainDualSecretRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainOfferSummary.java b/src/main/java/org/qortal/api/model/CrossChainOfferSummary.java new file mode 100644 index 000000000..bf71c2d29 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainOfferSummary.java @@ -0,0 +1,127 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import org.qortal.crosschain.AcctMode; +import org.qortal.data.crosschain.CrossChainTradeData; + +import io.swagger.v3.oas.annotations.media.Schema; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainOfferSummary { + + // Properties + + @Schema(description = "AT's Qortal address") + private String qortalAtAddress; + + @Schema(description = "AT creator's Qortal address") + private String qortalCreator; + + @Schema(description = "AT creator's ephemeral trading key-pair represented as Qortal address") + private String qortalCreatorTradeAddress; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long qortAmount; + + @Schema(description = "Bitcoin amount - DEPRECATED: use foreignAmount") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + @Deprecated + private long btcAmount; + + @Schema(description = "Foreign blockchain amount") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long foreignAmount; + + @Schema(description = "Suggested trade timeout (minutes)", example = "10080") + private int tradeTimeout; + + @Schema(description = "Current AT execution mode") + private AcctMode mode; + + private long timestamp; + + @Schema(description = "Trade partner's Qortal receiving address") + private String partnerQortalReceivingAddress; + + private String foreignBlockchain; + + private String acctName; + + protected CrossChainOfferSummary() { + /* For JAXB */ + } + + public CrossChainOfferSummary(CrossChainTradeData crossChainTradeData, long timestamp) { + this.qortalAtAddress = crossChainTradeData.qortalAtAddress; + this.qortalCreator = crossChainTradeData.qortalCreator; + this.qortalCreatorTradeAddress = crossChainTradeData.qortalCreatorTradeAddress; + this.qortAmount = crossChainTradeData.qortAmount; + this.foreignAmount = crossChainTradeData.expectedForeignAmount; + this.btcAmount = this.foreignAmount; // Duplicate for deprecated field + this.tradeTimeout = crossChainTradeData.tradeTimeout; + this.mode = crossChainTradeData.mode; + this.timestamp = timestamp; + this.partnerQortalReceivingAddress = crossChainTradeData.qortalPartnerReceivingAddress; + this.foreignBlockchain = crossChainTradeData.foreignBlockchain; + this.acctName = crossChainTradeData.acctName; + } + + public String getQortalAtAddress() { + return this.qortalAtAddress; + } + + public String getQortalCreator() { + return this.qortalCreator; + } + + public String getQortalCreatorTradeAddress() { + return this.qortalCreatorTradeAddress; + } + + public long getQortAmount() { + return this.qortAmount; + } + + public long getBtcAmount() { + return this.btcAmount; + } + + public long getForeignAmount() { + return this.foreignAmount; + } + + public int getTradeTimeout() { + return this.tradeTimeout; + } + + public AcctMode getMode() { + return this.mode; + } + + public long getTimestamp() { + return this.timestamp; + } + + public String getPartnerQortalReceivingAddress() { + return this.partnerQortalReceivingAddress; + } + + public String getForeignBlockchain() { + return this.foreignBlockchain; + } + + public String getAcctName() { + return this.acctName; + } + + // For debugging mostly + + public String toString() { + return String.format("%s: %s", this.qortalAtAddress, this.mode); + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainSecretRequest.java b/src/main/java/org/qortal/api/model/CrossChainSecretRequest.java index 998200221..2db475e52 100644 --- a/src/main/java/org/qortal/api/model/CrossChainSecretRequest.java +++ b/src/main/java/org/qortal/api/model/CrossChainSecretRequest.java @@ -8,15 +8,18 @@ @XmlAccessorType(XmlAccessType.FIELD) public class CrossChainSecretRequest { - @Schema(description = "Public key to match AT's 'recipient'", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") - public byte[] recipientPublicKey; + @Schema(description = "Private key to match AT's trade 'partner'", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") + public byte[] partnerPrivateKey; @Schema(description = "Qortal AT address") public String atAddress; - @Schema(description = "32-byte secret", example = "6gVbAXCVzJXAWwtAVGAfgAkkXpeXvPUwSciPmCfSfXJG") + @Schema(description = "Secret (32 bytes)", example = "FHMzten4he9jZ4HGb4297Utj6F5g2w7serjq2EnAg2s1") public byte[] secret; + @Schema(description = "Qortal address for receiving QORT from AT") + public String receivingAddress; + public CrossChainSecretRequest() { } diff --git a/src/main/java/org/qortal/api/model/CrossChainTradeRequest.java b/src/main/java/org/qortal/api/model/CrossChainTradeRequest.java index 32737dd53..1afd72901 100644 --- a/src/main/java/org/qortal/api/model/CrossChainTradeRequest.java +++ b/src/main/java/org/qortal/api/model/CrossChainTradeRequest.java @@ -8,14 +8,14 @@ @XmlAccessorType(XmlAccessType.FIELD) public class CrossChainTradeRequest { - @Schema(description = "AT creator's public key", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") - public byte[] creatorPublicKey; + @Schema(description = "AT creator's 'trade' public key", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") + public byte[] tradePublicKey; @Schema(description = "Qortal AT address") public String atAddress; - @Schema(description = "Qortal address for trade partner/recipient") - public String recipient; + @Schema(description = "Signature of trading partner's 'offer' MESSAGE transaction") + public byte[] messageTransactionSignature; public CrossChainTradeRequest() { } diff --git a/src/main/java/org/qortal/api/model/CrossChainTradeSummary.java b/src/main/java/org/qortal/api/model/CrossChainTradeSummary.java new file mode 100644 index 000000000..edc137c09 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainTradeSummary.java @@ -0,0 +1,67 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import org.qortal.data.crosschain.CrossChainTradeData; + +import io.swagger.v3.oas.annotations.media.Schema; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainTradeSummary { + + private long tradeTimestamp; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long qortAmount; + + @Deprecated + @Schema(description = "DEPRECATED: use foreignAmount instead") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long btcAmount; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long foreignAmount; + + private String atAddress; + + private String sellerAddress; + + private String buyerReceivingAddress; + + protected CrossChainTradeSummary() { + /* For JAXB */ + } + + public CrossChainTradeSummary(CrossChainTradeData crossChainTradeData, long timestamp) { + this.tradeTimestamp = timestamp; + this.qortAmount = crossChainTradeData.qortAmount; + this.foreignAmount = crossChainTradeData.expectedForeignAmount; + this.btcAmount = this.foreignAmount; + this.sellerAddress = crossChainTradeData.qortalCreator; + this.buyerReceivingAddress = crossChainTradeData.qortalPartnerReceivingAddress; + this.atAddress = crossChainTradeData.qortalAtAddress; + } + + public long getTradeTimestamp() { + return this.tradeTimestamp; + } + + public long getQortAmount() { + return this.qortAmount; + } + + public long getBtcAmount() { + return this.btcAmount; + } + + public long getForeignAmount() { return this.foreignAmount; } + + public String getAtAddress() { return this.atAddress; } + + public String getSellerAddress() { return this.sellerAddress; } + + public String getBuyerReceivingAddressAddress() { return this.buyerReceivingAddress; } +} diff --git a/src/main/java/org/qortal/api/model/ListRequest.java b/src/main/java/org/qortal/api/model/ListRequest.java new file mode 100644 index 000000000..c5ebb48a9 --- /dev/null +++ b/src/main/java/org/qortal/api/model/ListRequest.java @@ -0,0 +1,18 @@ +package org.qortal.api.model; + +import io.swagger.v3.oas.annotations.media.Schema; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.List; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ListRequest { + + @Schema(description = "A list of items") + public List items; + + public ListRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/NodeInfo.java b/src/main/java/org/qortal/api/model/NodeInfo.java index 86ed6971d..16a4df75b 100644 --- a/src/main/java/org/qortal/api/model/NodeInfo.java +++ b/src/main/java/org/qortal/api/model/NodeInfo.java @@ -11,6 +11,7 @@ public class NodeInfo { public String buildVersion; public long buildTimestamp; public String nodeId; + public boolean isTestNet; public NodeInfo() { } diff --git a/src/main/java/org/qortal/api/model/NodeStatus.java b/src/main/java/org/qortal/api/model/NodeStatus.java index 9e50df24b..ccc1eb011 100644 --- a/src/main/java/org/qortal/api/model/NodeStatus.java +++ b/src/main/java/org/qortal/api/model/NodeStatus.java @@ -4,6 +4,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.network.Network; @XmlAccessorType(XmlAccessType.FIELD) @@ -20,17 +21,14 @@ public class NodeStatus { public final int height; public NodeStatus() { - isMintingPossible = Controller.getInstance().isMintingPossible(); - isSynchronizing = Controller.getInstance().isSynchronizing(); + this.isMintingPossible = Controller.getInstance().isMintingPossible(); - if (isSynchronizing) - syncPercent = Controller.getInstance().getSyncPercent(); - else - syncPercent = null; + this.syncPercent = Synchronizer.getInstance().getSyncPercent(); + this.isSynchronizing = this.syncPercent != null; - numberOfConnections = Network.getInstance().getHandshakedPeers().size(); + this.numberOfConnections = Network.getInstance().getHandshakedPeers().size(); - height = Controller.getInstance().getChainHeight(); + this.height = Controller.getInstance().getChainHeight(); } } diff --git a/src/main/java/org/qortal/api/model/PeersSummary.java b/src/main/java/org/qortal/api/model/PeersSummary.java new file mode 100644 index 000000000..28788550c --- /dev/null +++ b/src/main/java/org/qortal/api/model/PeersSummary.java @@ -0,0 +1,15 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class PeersSummary { + + public int inboundConnections; + public int outboundConnections; + + public PeersSummary() { + } + +} diff --git a/src/main/java/org/qortal/api/model/SimpleForeignTransaction.java b/src/main/java/org/qortal/api/model/SimpleForeignTransaction.java new file mode 100644 index 000000000..acc1120f7 --- /dev/null +++ b/src/main/java/org/qortal/api/model/SimpleForeignTransaction.java @@ -0,0 +1,157 @@ +package org.qortal.api.model; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class SimpleForeignTransaction { + + public static class AddressAmount { + public final String address; + public final long amount; + + protected AddressAmount() { + /* For JAXB */ + this.address = null; + this.amount = 0; + } + + public AddressAmount(String address, long amount) { + this.address = address; + this.amount = amount; + } + } + + private String txHash; + private long timestamp; + + private List inputs; + + public static class Output { + public final List addresses; + public final long amount; + + protected Output() { + /* For JAXB */ + this.addresses = null; + this.amount = 0; + } + + public Output(List addresses, long amount) { + this.addresses = addresses; + this.amount = amount; + } + } + private List outputs; + + private long totalAmount; + private long fees; + + private Boolean isSentNotReceived; + + protected SimpleForeignTransaction() { + /* For JAXB */ + } + + private SimpleForeignTransaction(Builder builder) { + this.txHash = builder.txHash; + this.timestamp = builder.timestamp; + this.inputs = Collections.unmodifiableList(builder.inputs); + this.outputs = Collections.unmodifiableList(builder.outputs); + + Objects.requireNonNull(this.txHash); + if (timestamp <= 0) + throw new IllegalArgumentException("timestamp must be positive"); + + long totalGrossAmount = this.inputs.stream().map(addressAmount -> addressAmount.amount).reduce(0L, Long::sum); + this.totalAmount = this.outputs.stream().map(addressAmount -> addressAmount.amount).reduce(0L, Long::sum); + + this.fees = totalGrossAmount - this.totalAmount; + + this.isSentNotReceived = builder.isSentNotReceived; + } + + public String getTxHash() { + return this.txHash; + } + + public long getTimestamp() { + return this.timestamp; + } + + public List getInputs() { + return this.inputs; + } + + public List getOutputs() { + return this.outputs; + } + + public long getTotalAmount() { + return this.totalAmount; + } + + public long getFees() { + return this.fees; + } + + public Boolean isSentNotReceived() { + return this.isSentNotReceived; + } + + public static class Builder { + private String txHash; + private long timestamp; + private List inputs = new ArrayList<>(); + private List outputs = new ArrayList<>(); + private Boolean isSentNotReceived; + + public Builder txHash(String txHash) { + this.txHash = Objects.requireNonNull(txHash); + return this; + } + + public Builder timestamp(long timestamp) { + if (timestamp <= 0) + throw new IllegalArgumentException("timestamp must be positive"); + + this.timestamp = timestamp; + return this; + } + + public Builder input(String address, long amount) { + Objects.requireNonNull(address); + if (amount < 0) + throw new IllegalArgumentException("amount must be zero or positive"); + + AddressAmount input = new AddressAmount(address, amount); + inputs.add(input); + return this; + } + + public Builder output(List addresses, long amount) { + Objects.requireNonNull(addresses); + if (amount < 0) + throw new IllegalArgumentException("amount must be zero or positive"); + + Output output = new Output(addresses, amount); + outputs.add(output); + return this; + } + + public Builder isSentNotReceived(Boolean isSentNotReceived) { + this.isSentNotReceived = isSentNotReceived; + return this; + } + + public SimpleForeignTransaction build() { + return new SimpleForeignTransaction(this); + } + } + +} diff --git a/src/main/java/org/qortal/api/model/crosschain/BitcoinSendRequest.java b/src/main/java/org/qortal/api/model/crosschain/BitcoinSendRequest.java new file mode 100644 index 000000000..86d3d7c84 --- /dev/null +++ b/src/main/java/org/qortal/api/model/crosschain/BitcoinSendRequest.java @@ -0,0 +1,29 @@ +package org.qortal.api.model.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class BitcoinSendRequest { + + @Schema(description = "Bitcoin BIP32 extended private key", example = "tprv___________________________________________________________________________________________________________") + public String xprv58; + + @Schema(description = "Recipient's Bitcoin address ('legacy' P2PKH only)", example = "1BitcoinEaterAddressDontSendf59kuE") + public String receivingAddress; + + @Schema(description = "Amount of BTC to send", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long bitcoinAmount; + + @Schema(description = "Transaction fee per byte (optional). Default is 0.00000100 BTC (100 sats) per byte", example = "0.00000100", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public Long feePerByte; + + public BitcoinSendRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/crosschain/DogecoinSendRequest.java b/src/main/java/org/qortal/api/model/crosschain/DogecoinSendRequest.java new file mode 100644 index 000000000..88740058f --- /dev/null +++ b/src/main/java/org/qortal/api/model/crosschain/DogecoinSendRequest.java @@ -0,0 +1,29 @@ +package org.qortal.api.model.crosschain; + +import io.swagger.v3.oas.annotations.media.Schema; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +@XmlAccessorType(XmlAccessType.FIELD) +public class DogecoinSendRequest { + + @Schema(description = "Dogecoin BIP32 extended private key", example = "tprv___________________________________________________________________________________________________________") + public String xprv58; + + @Schema(description = "Recipient's Dogecoin address ('legacy' P2PKH only)", example = "DoGecoinEaterAddressDontSendhLfzKD") + public String receivingAddress; + + @Schema(description = "Amount of DOGE to send", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long dogecoinAmount; + + @Schema(description = "Transaction fee per byte (optional). Default is 0.00000100 DOGE (100 sats) per byte", example = "0.00000100", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public Long feePerByte; + + public DogecoinSendRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/crosschain/LitecoinSendRequest.java b/src/main/java/org/qortal/api/model/crosschain/LitecoinSendRequest.java new file mode 100644 index 000000000..5f2157403 --- /dev/null +++ b/src/main/java/org/qortal/api/model/crosschain/LitecoinSendRequest.java @@ -0,0 +1,29 @@ +package org.qortal.api.model.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class LitecoinSendRequest { + + @Schema(description = "Litecoin BIP32 extended private key", example = "tprv___________________________________________________________________________________________________________") + public String xprv58; + + @Schema(description = "Recipient's Litecoin address ('legacy' P2PKH only)", example = "LiTecoinEaterAddressDontSendhLfzKD") + public String receivingAddress; + + @Schema(description = "Amount of LTC to send", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long litecoinAmount; + + @Schema(description = "Transaction fee per byte (optional). Default is 0.00000100 LTC (100 sats) per byte", example = "0.00000100", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public Long feePerByte; + + public LitecoinSendRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/crosschain/TradeBotCreateRequest.java b/src/main/java/org/qortal/api/model/crosschain/TradeBotCreateRequest.java new file mode 100644 index 000000000..1f96488ea --- /dev/null +++ b/src/main/java/org/qortal/api/model/crosschain/TradeBotCreateRequest.java @@ -0,0 +1,46 @@ +package org.qortal.api.model.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import org.qortal.crosschain.SupportedBlockchain; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class TradeBotCreateRequest { + + @Schema(description = "Trade creator's public key", example = "2zR1WFsbM7akHghqSCYKBPk6LDP8aKiQSRS1FrwoLvoB") + public byte[] creatorPublicKey; + + @Schema(description = "QORT amount paid out on successful trade", example = "80.40000000", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long qortAmount; + + @Schema(description = "QORT amount funding AT, including covering AT execution fees", example = "80.50000000", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long fundingQortAmount; + + @Deprecated + @Schema(description = "Bitcoin amount wanted in return. DEPRECATED: use foreignAmount instead", example = "0.00864200", type = "number", hidden = true) + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public Long bitcoinAmount; + + @Schema(description = "Foreign blockchain. Note: default (BITCOIN) to be removed in the future", example = "BITCOIN", implementation = SupportedBlockchain.class) + public SupportedBlockchain foreignBlockchain; + + @Schema(description = "Foreign blockchain amount wanted in return", example = "0.00864200", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public Long foreignAmount; + + @Schema(description = "Suggested trade timeout (minutes)", example = "10080") + public int tradeTimeout; + + @Schema(description = "Foreign blockchain address for receiving", example = "1BitcoinEaterAddressDontSendf59kuE") + public String receivingAddress; + + public TradeBotCreateRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/crosschain/TradeBotRespondRequest.java b/src/main/java/org/qortal/api/model/crosschain/TradeBotRespondRequest.java new file mode 100644 index 000000000..ecc8ed6fd --- /dev/null +++ b/src/main/java/org/qortal/api/model/crosschain/TradeBotRespondRequest.java @@ -0,0 +1,29 @@ +package org.qortal.api.model.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class TradeBotRespondRequest { + + @Schema(description = "Qortal AT address", example = "Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + public String atAddress; + + @Deprecated + @Schema(description = "Bitcoin BIP32 extended private key. DEPRECATED: use foreignKey instead", hidden = true, + example = "xprv___________________________________________________________________________________________________________") + public String xprv58; + + @Schema(description = "Foreign blockchain private key, e.g. BIP32 'm' key for Bitcoin/Litecoin starting with 'xprv'", + example = "xprv___________________________________________________________________________________________________________") + public String foreignKey; + + @Schema(description = "Qortal address for receiving QORT from AT", example = "Qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq") + public String receivingAddress; + + public TradeBotRespondRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/resource/AddressesResource.java b/src/main/java/org/qortal/api/resource/AddressesResource.java index 39b4bd71f..39c76cf46 100644 --- a/src/main/java/org/qortal/api/resource/AddressesResource.java +++ b/src/main/java/org/qortal/api/resource/AddressesResource.java @@ -7,18 +7,16 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; import java.math.BigDecimal; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.QueryParam; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; @@ -37,6 +35,7 @@ import org.qortal.data.account.AccountData; import org.qortal.data.account.RewardShareData; import org.qortal.data.network.OnlineAccountData; +import org.qortal.data.network.OnlineAccountLevel; import org.qortal.data.transaction.PublicizeTransactionData; import org.qortal.data.transaction.RewardShareTransactionData; import org.qortal.data.transaction.TransactionData; @@ -179,6 +178,66 @@ public List getOnlineAccounts() { } } + @GET + @Path("/online/levels") + @Operation( + summary = "Return currently 'online' accounts counts, grouped by level", + responses = { + @ApiResponse( + description = "online accounts", + content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = ApiOnlineAccount.class))) + ) + } + ) + @ApiErrors({ApiError.PUBLIC_KEY_NOT_FOUND, ApiError.REPOSITORY_ISSUE}) + public List getOnlineAccountsByLevel() { + List onlineAccounts = Controller.getInstance().getOnlineAccounts(); + + try (final Repository repository = RepositoryManager.getRepository()) { + List onlineAccountLevels = new ArrayList<>(); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + try { + final int minterLevel = Account.getRewardShareEffectiveMintingLevelIncludingLevelZero(repository, onlineAccountData.getPublicKey()); + + OnlineAccountLevel onlineAccountLevel = onlineAccountLevels.stream() + .filter(a -> a.getLevel() == minterLevel) + .findFirst().orElse(null); + + // Note: I don't think we can use the level as the List index here because there will be gaps. + // So we are forced to manually look up the existing item each time. + // There's probably a nice shorthand java way of doing this, but this approach gets the same result. + + if (onlineAccountLevel == null) { + // No entry exists for this level yet, so create one + onlineAccountLevel = new OnlineAccountLevel(minterLevel, 1); + onlineAccountLevels.add(onlineAccountLevel); + } + else { + // Already exists - so increment the count + int existingCount = onlineAccountLevel.getCount(); + onlineAccountLevel.setCount(++existingCount); + + // Then replace the existing item + int index = onlineAccountLevels.indexOf(onlineAccountLevel); + onlineAccountLevels.set(index, onlineAccountLevel); + } + + } catch (DataException e) { + continue; + } + } + + // Sort by level + onlineAccountLevels.sort(Comparator.comparingInt(OnlineAccountLevel::getLevel)); + + return onlineAccountLevels; + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + @GET @Path("/balance/{address}") @Operation( @@ -473,7 +532,8 @@ public String publicize(PublicizeTransactionData transactionData) { } ) @ApiErrors({ApiError.TRANSACTION_INVALID, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE}) - public String computePublicize(String rawBytes58) { + @SecurityRequirement(name = "apiKey") + public String computePublicize(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String rawBytes58) { Security.checkApiCallAllowed(request); try (final Repository repository = RepositoryManager.getRepository()) { diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index 9b765f402..7f4a38063 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -8,6 +8,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; import java.io.IOException; @@ -21,33 +22,29 @@ import java.time.ZoneOffset; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.QueryParam; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.RollingFileAppender; +import org.checkerframework.checker.units.qual.A; import org.qortal.account.Account; import org.qortal.account.PrivateKeyAccount; -import org.qortal.api.ApiError; -import org.qortal.api.ApiErrors; -import org.qortal.api.ApiException; -import org.qortal.api.ApiExceptionFactory; -import org.qortal.api.Security; +import org.qortal.api.*; import org.qortal.api.model.ActivitySummary; import org.qortal.api.model.NodeInfo; import org.qortal.api.model.NodeStatus; import org.qortal.block.BlockChain; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.controller.Synchronizer.SynchronizationResult; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; @@ -57,6 +54,7 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; import org.qortal.utils.Base58; import org.qortal.utils.NTP; @@ -66,6 +64,8 @@ @Tag(name = "Admin") public class AdminResource { + private static final Logger LOGGER = LogManager.getLogger(AdminResource.class); + private static final int MAX_LOG_LINES = 500; @Context @@ -75,7 +75,8 @@ public class AdminResource { @Path("/unused") @Parameter(in = ParameterIn.PATH, name = "assetid", description = "Asset ID, 0 is native coin", schema = @Schema(type = "integer")) @Parameter(in = ParameterIn.PATH, name = "otherassetid", description = "Asset ID, 0 is native coin", schema = @Schema(type = "integer")) - @Parameter(in = ParameterIn.PATH, name = "address", description = "an account address", example = "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v") + @Parameter(in = ParameterIn.PATH, name = "address", description = "An account address", example = "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v") + @Parameter(in = ParameterIn.PATH, name = "path", description = "Local path to folder containing the files", schema = @Schema(type = "String", defaultValue = "/Users/user/Documents/MyStaticWebsite")) @Parameter(in = ParameterIn.QUERY, name = "count", description = "Maximum number of entries to return, 0 means none", schema = @Schema(type = "integer", defaultValue = "20")) @Parameter(in = ParameterIn.QUERY, name = "limit", description = "Maximum number of entries to return, 0 means unlimited", schema = @Schema(type = "integer", defaultValue = "20")) @Parameter(in = ParameterIn.QUERY, name = "offset", description = "Starting entry in results, 0 is first entry", schema = @Schema(type = "integer")) @@ -118,6 +119,7 @@ public NodeInfo info() { nodeInfo.buildVersion = Controller.getInstance().getVersionString(); nodeInfo.buildTimestamp = Controller.getInstance().getBuildTimestamp(); nodeInfo.nodeId = Network.getInstance().getOurNodeId(); + nodeInfo.isTestNet = Settings.getInstance().isTestNet(); return nodeInfo; } @@ -133,8 +135,6 @@ public NodeInfo info() { } ) public NodeStatus status() { - Security.checkApiCallAllowed(request); - NodeStatus nodeStatus = new NodeStatus(); return nodeStatus; @@ -152,7 +152,8 @@ public NodeStatus status() { ) } ) - public String shutdown() { + @SecurityRequirement(name = "apiKey") + public String shutdown(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { Security.checkApiCallAllowed(request); new Thread(() -> { @@ -180,7 +181,10 @@ public String shutdown() { } ) @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public ActivitySummary summary() { + @SecurityRequirement(name = "apiKey") + public ActivitySummary summary(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + ActivitySummary summary = new ActivitySummary(); LocalDate date = LocalDate.now(); @@ -192,16 +196,13 @@ public ActivitySummary summary() { int startHeight = repository.getBlockRepository().getHeightFromTimestamp(start); int endHeight = repository.getBlockRepository().getBlockchainHeight(); - summary.blockCount = endHeight - startHeight; - - summary.transactionCountByType = repository.getTransactionRepository().getTransactionSummary(startHeight + 1, endHeight); + summary.setBlockCount(endHeight - startHeight); - for (Integer count : summary.transactionCountByType.values()) - summary.transactionCount += count; + summary.setTransactionCountByType(repository.getTransactionRepository().getTransactionSummary(startHeight + 1, endHeight)); - summary.assetsIssued = repository.getAssetRepository().getRecentAssetIds(start).size(); + summary.setAssetsIssued(repository.getAssetRepository().getRecentAssetIds(start).size()); - summary.namesRegistered = repository.getNameRepository().getRecentNames(start).size(); + summary.setNamesRegistered (repository.getNameRepository().getRecentNames(start).size()); return summary; } catch (DataException e) { @@ -209,6 +210,30 @@ public ActivitySummary summary() { } } + @GET + @Path("/enginestats") + @Operation( + summary = "Fetch statistics snapshot for core engine", + responses = { + @ApiResponse( + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + array = @ArraySchema( + schema = @Schema( + implementation = Controller.StatsSnapshot.class + ) + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public Controller.StatsSnapshot getEngineStats(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + + return Controller.getInstance().getStatsSnapshot(); + } + @GET @Path("/mintingaccounts") @Operation( @@ -222,7 +247,6 @@ public ActivitySummary summary() { ) @ApiErrors({ApiError.REPOSITORY_ISSUE}) public List getMintingAccounts() { - Security.checkApiCallAllowed(request); try (final Repository repository = RepositoryManager.getRepository()) { List mintingAccounts = repository.getAccountRepository().getMintingAccounts(); @@ -267,7 +291,8 @@ public List getMintingAccounts() { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.REPOSITORY_ISSUE, ApiError.CANNOT_MINT}) - public String addMintingAccount(String seed58) { + @SecurityRequirement(name = "apiKey") + public String addMintingAccount(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String seed58) { Security.checkApiCallAllowed(request); try (final Repository repository = RepositoryManager.getRepository()) { @@ -290,6 +315,7 @@ public String addMintingAccount(String seed58) { repository.getAccountRepository().save(mintingAccountData); repository.saveChanges(); + repository.exportNodeLocalData();//after adding new minting account let's persist it to the backup MintingAccounts.json } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e); } catch (DataException e) { @@ -302,13 +328,13 @@ public String addMintingAccount(String seed58) { @DELETE @Path("/mintingaccounts") @Operation( - summary = "Remove account/reward-share from use by BlockMinter, using private key", + summary = "Remove account/reward-share from use by BlockMinter, using public or private key", requestBody = @RequestBody( required = true, content = @Content( mediaType = MediaType.TEXT_PLAIN, schema = @Schema( - type = "string", example = "private key" + type = "string", example = "public or private key" ) ) ), @@ -319,16 +345,18 @@ public String addMintingAccount(String seed58) { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.REPOSITORY_ISSUE}) - public String deleteMintingAccount(String seed58) { + @SecurityRequirement(name = "apiKey") + public String deleteMintingAccount(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) { Security.checkApiCallAllowed(request); try (final Repository repository = RepositoryManager.getRepository()) { - byte[] seed = Base58.decode(seed58.trim()); + byte[] key = Base58.decode(key58.trim()); - if (repository.getAccountRepository().delete(seed) == 0) + if (repository.getAccountRepository().delete(key) == 0) return "false"; repository.saveChanges(); + repository.exportNodeLocalData();//after removing new minting account let's persist it to the backup MintingAccounts.json } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e); } catch (DataException e) { @@ -418,7 +446,8 @@ public String fetchLogs(@Parameter( } ) @ApiErrors({ApiError.INVALID_HEIGHT, ApiError.REPOSITORY_ISSUE}) - public String orphan(String targetHeightString) { + @SecurityRequirement(name = "apiKey") + public String orphan(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String targetHeightString) { Security.checkApiCallAllowed(request); try { @@ -427,6 +456,23 @@ public String orphan(String targetHeightString) { if (targetHeight <= 0 || targetHeight > Controller.getInstance().getChainHeight()) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT); + // Make sure we're not orphaning as far back as the archived blocks + // FUTURE: we could support this by first importing earlier blocks from the archive + if (Settings.getInstance().isTopOnly() || + Settings.getInstance().isArchiveEnabled()) { + + try (final Repository repository = RepositoryManager.getRepository()) { + // Find the first unarchived block + int oldestBlock = repository.getBlockArchiveRepository().getBlockArchiveHeight(); + // Add some extra blocks just in case we're currently archiving/pruning + oldestBlock += 100; + if (targetHeight <= oldestBlock) { + LOGGER.info("Unable to orphan beyond block {} because it is archived", oldestBlock); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT); + } + } + } + if (BlockChain.orphan(targetHeight)) return "true"; else @@ -435,8 +481,6 @@ public String orphan(String targetHeightString) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } catch (NumberFormatException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT); - } catch (ApiException e) { - throw e; } } @@ -461,7 +505,8 @@ public String orphan(String targetHeightString) { } ) @ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE}) - public String forceSync(String targetPeerAddress) { + @SecurityRequirement(name = "apiKey") + public String forceSync(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String targetPeerAddress) { Security.checkApiCallAllowed(request); try { @@ -483,7 +528,7 @@ public String forceSync(String targetPeerAddress) { SynchronizationResult syncResult; try { do { - syncResult = Controller.getInstance().actuallySynchronize(targetPeer, true); + syncResult = Synchronizer.getInstance().actuallySynchronize(targetPeer, true); } while (syncResult == SynchronizationResult.OK); } finally { blockchainLock.unlock(); @@ -492,8 +537,6 @@ public String forceSync(String targetPeerAddress) { return syncResult.name(); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } catch (ApiException e) { - throw e; } catch (UnknownHostException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } catch (InterruptedException e) { @@ -501,4 +544,223 @@ public String forceSync(String targetPeerAddress) { } } + @GET + @Path("/repository/data") + @Operation( + summary = "Export sensitive/node-local data from repository.", + description = "Exports data to .json files on local machine" + ) + @ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String exportRepository(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + repository.exportNodeLocalData(); + return "true"; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST + @Path("/repository/data") + @Operation( + summary = "Import data into repository.", + description = "Imports data from file on local machine. Filename is forced to 'qortal-backup/TradeBotStates.json' if apiKey is not set.", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "qortal-backup/TradeBotStates.json" + ) + ) + ), + responses = { + @ApiResponse( + description = "\"true\"", + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String importRepository(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String filename) { + Security.checkApiCallAllowed(request); + + // Hard-coded because it's too dangerous to allow user-supplied filenames in weaker security contexts + if (Settings.getInstance().getApiKey() == null) + filename = "qortal-backup/TradeBotStates.json"; + + try (final Repository repository = RepositoryManager.getRepository()) { + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + + blockchainLock.lockInterruptibly(); + + try { + repository.importDataFromFile(filename); + repository.saveChanges(); + + return "true"; + + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e); + + } finally { + blockchainLock.unlock(); + } + } catch (InterruptedException e) { + // We couldn't lock blockchain to perform import + return "false"; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST + @Path("/repository/checkpoint") + @Operation( + summary = "Checkpoint data in repository.", + description = "Forces repository to checkpoint uncommitted writes.", + responses = { + @ApiResponse( + description = "\"true\"", + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String checkpointRepository(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + + RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); + + return "true"; + } + + @POST + @Path("/repository/backup") + @Operation( + summary = "Perform online backup of repository.", + responses = { + @ApiResponse( + description = "\"true\"", + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String backupRepository(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + + blockchainLock.lockInterruptibly(); + + try { + // Timeout if the database isn't ready for backing up after 60 seconds + long timeout = 60 * 1000L; + repository.backup(true, "backup", timeout); + repository.saveChanges(); + + return "true"; + } finally { + blockchainLock.unlock(); + } + } catch (InterruptedException | TimeoutException e) { + // We couldn't lock blockchain to perform backup + return "false"; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @DELETE + @Path("/repository") + @Operation( + summary = "Perform maintenance on repository.", + description = "Requires enough free space to rebuild repository. This will pause your node for a while." + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public void performRepositoryMaintenance(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + + blockchainLock.lockInterruptibly(); + + try { + // Timeout if the database isn't ready to start after 60 seconds + long timeout = 60 * 1000L; + repository.performPeriodicMaintenance(timeout); + } finally { + blockchainLock.unlock(); + } + } catch (InterruptedException e) { + // No big deal + } catch (DataException | TimeoutException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + + @POST + @Path("/apikey/generate") + @Operation( + summary = "Generate an API key", + description = "This request is unauthenticated if no API key has been generated yet. " + + "If an API key already exists, it needs to be passed as a header and this endpoint " + + "will then generate a new key which replaces the existing one.", + responses = { + @ApiResponse( + description = "API key string", + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String generateApiKey(@HeaderParam(Security.API_KEY_HEADER) String apiKeyHeader) { + ApiKey apiKey = Security.getApiKey(request); + + // If the API key is already generated, we need to authenticate this request + if (apiKey.generated() && apiKey.exists()) { + Security.checkApiCallAllowed(request); + } + + // Not generated yet - so we are safe to generate one + // FUTURE: we may want to restrict this to local/loopback only? + + try { + apiKey.generate(); + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Unable to generate API key"); + } + + return apiKey.toString(); + } + + @GET + @Path("/apikey/test") + @Operation( + summary = "Test an API key", + responses = { + @ApiResponse( + description = "true if authenticated", + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String testApiKey(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + + return "true"; + } + } diff --git a/src/main/java/org/qortal/api/resource/ApiDefinition.java b/src/main/java/org/qortal/api/resource/ApiDefinition.java index ae7de00c8..f9ec7459b 100644 --- a/src/main/java/org/qortal/api/resource/ApiDefinition.java +++ b/src/main/java/org/qortal/api/resource/ApiDefinition.java @@ -1,11 +1,17 @@ package org.qortal.api.resource; import io.swagger.v3.oas.annotations.OpenAPIDefinition; +import io.swagger.v3.oas.annotations.enums.SecuritySchemeIn; +import io.swagger.v3.oas.annotations.enums.SecuritySchemeType; import io.swagger.v3.oas.annotations.extensions.Extension; import io.swagger.v3.oas.annotations.extensions.ExtensionProperty; import io.swagger.v3.oas.annotations.info.Info; +import io.swagger.v3.oas.annotations.security.SecurityScheme; +import io.swagger.v3.oas.annotations.security.SecuritySchemes; import io.swagger.v3.oas.annotations.tags.Tag; +import org.qortal.api.Security; + @OpenAPIDefinition( info = @Info( title = "Qortal API", description = "NOTE: byte-arrays are encoded in Base58" ), tags = { @@ -30,5 +36,9 @@ }) } ) +@SecuritySchemes({ + @SecurityScheme(name = "basicAuth", type = SecuritySchemeType.HTTP, scheme = "basic"), + @SecurityScheme(name = "apiKey", type = SecuritySchemeType.APIKEY, in = SecuritySchemeIn.HEADER, paramName = Security.API_KEY_HEADER) +}) public class ApiDefinition { } \ No newline at end of file diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 266043189..ba39825c7 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1,5 +1,6 @@ package org.qortal.api.resource; +import com.google.common.primitives.Bytes; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.ArraySchema; @@ -7,28 +8,41 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.QueryParam; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import org.qortal.api.ApiError; -import org.qortal.api.ApiErrors; -import org.qortal.api.ApiException; -import org.qortal.api.ApiExceptionFactory; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bouncycastle.util.encoders.Base64; +import org.qortal.api.*; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; +import org.qortal.arbitrary.*; +import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.Controller; +import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; +import org.qortal.data.account.AccountData; +import org.qortal.data.arbitrary.ArbitraryResourceInfo; +import org.qortal.data.arbitrary.ArbitraryResourceNameInfo; +import org.qortal.data.arbitrary.ArbitraryResourceStatus; +import org.qortal.data.naming.NameData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; -import org.qortal.data.transaction.ArbitraryTransactionData.DataType; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -39,15 +53,217 @@ import org.qortal.transaction.Transaction.ValidationResult; import org.qortal.transform.TransformationException; import org.qortal.transform.transaction.ArbitraryTransactionTransformer; +import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.utils.Base58; +import org.qortal.utils.ZipUtils; @Path("/arbitrary") @Tag(name = "Arbitrary") public class ArbitraryResource { - @Context - HttpServletRequest request; - + private static final Logger LOGGER = LogManager.getLogger(ArbitraryResource.class); + + @Context HttpServletRequest request; + @Context HttpServletResponse response; + @Context ServletContext context; + + @GET + @Path("/resources") + @Operation( + summary = "List arbitrary resources available on chain, optionally filtered by service and identifier", + description = "- If the identifier parameter is missing or empty, it will return an unfiltered list of all possible identifiers.\n" + + "- If an identifier is specified, only resources with a matching identifier will be returned.\n" + + "- If default is set to true, only resources without identifiers will be returned.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class)) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + public List getResources( + @QueryParam("service") Service service, + @QueryParam("identifier") String identifier, + @Parameter(description = "Default resources (without identifiers) only") @QueryParam("default") Boolean defaultResource, + @Parameter(ref = "limit") @QueryParam("limit") Integer limit, + @Parameter(ref = "offset") @QueryParam("offset") Integer offset, + @Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse, + @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus) { + + try (final Repository repository = RepositoryManager.getRepository()) { + + // Treat empty identifier as null + if (identifier != null && identifier.isEmpty()) { + identifier = null; + } + + // Ensure that "default" and "identifier" parameters cannot coexist + boolean defaultRes = Boolean.TRUE.equals(defaultResource); + if (defaultRes == true && identifier != null) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "identifier cannot be specified when requesting a default resource"); + } + + List resources = repository.getArbitraryRepository() + .getArbitraryResources(service, identifier, null, defaultRes, limit, offset, reverse); + + if (resources == null) { + return new ArrayList<>(); + } + + if (includeStatus != null && includeStatus == true) { + resources = this.addStatusToResources(resources); + } + + return resources; + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @GET + @Path("/resources/search") + @Operation( + summary = "Search arbitrary resources available on chain, optionally filtered by service.\n" + + "If default is set to true, only resources without identifiers will be returned.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class)) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + public List searchResources( + @QueryParam("service") Service service, + @QueryParam("query") String query, + @Parameter(description = "Default resources (without identifiers) only") @QueryParam("default") Boolean defaultResource, + @Parameter(ref = "limit") @QueryParam("limit") Integer limit, + @Parameter(ref = "offset") @QueryParam("offset") Integer offset, + @Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse, + @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus) { + + try (final Repository repository = RepositoryManager.getRepository()) { + + boolean defaultRes = Boolean.TRUE.equals(defaultResource); + + List resources = repository.getArbitraryRepository() + .searchArbitraryResources(service, query, defaultRes, limit, offset, reverse); + + if (resources == null) { + return new ArrayList<>(); + } + + if (includeStatus != null && includeStatus == true) { + resources = this.addStatusToResources(resources); + } + + return resources; + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @GET + @Path("/resources/names") + @Operation( + summary = "List arbitrary resources available on chain, grouped by creator's name", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class)) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + public List getResourcesGroupedByName( + @QueryParam("service") Service service, + @QueryParam("identifier") String identifier, + @Parameter(description = "Default resources (without identifiers) only") @QueryParam("default") Boolean defaultResource, + @Parameter(ref = "limit") @QueryParam("limit") Integer limit, + @Parameter(ref = "offset") @QueryParam("offset") Integer offset, + @Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse, + @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus) { + + try (final Repository repository = RepositoryManager.getRepository()) { + + // Treat empty identifier as null + if (identifier != null && identifier.isEmpty()) { + identifier = null; + } + + // Ensure that "default" and "identifier" parameters cannot coexist + boolean defaultRes = Boolean.TRUE.equals(defaultResource); + if (defaultRes == true && identifier != null) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "identifier cannot be specified when requesting a default resource"); + } + + List creatorNames = repository.getArbitraryRepository() + .getArbitraryResourceCreatorNames(service, identifier, defaultRes, limit, offset, reverse); + + for (ArbitraryResourceNameInfo creatorName : creatorNames) { + String name = creatorName.name; + if (name != null) { + List resources = repository.getArbitraryRepository() + .getArbitraryResources(service, identifier, name, defaultRes, null, null, reverse); + + if (includeStatus != null && includeStatus == true) { + resources = this.addStatusToResources(resources); + } + creatorName.resources = resources; + } + } + + return creatorNames; + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @GET + @Path("/resource/status/{service}/{name}") + @Operation( + summary = "Get status of arbitrary resource with supplied service and name", + description = "If build is set to true, the resource will be built synchronously before returning the status.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceStatus.class)) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public ArbitraryResourceStatus getDefaultResourceStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") Service service, + @PathParam("name") String name, + @QueryParam("build") Boolean build) { + + Security.requirePriorAuthorizationOrApiKey(request, name, service, null); + return this.getStatus(service, name, null, build); + } + + @GET + @Path("/resource/status/{service}/{name}/{identifier}") + @Operation( + summary = "Get status of arbitrary resource with supplied service, name and identifier", + description = "If build is set to true, the resource will be built synchronously before returning the status.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceStatus.class)) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public ArbitraryResourceStatus getResourceStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") Service service, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @QueryParam("build") Boolean build) { + + Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier); + return this.getStatus(service, name, identifier, build); + } + + @GET @Path("/search") @Operation( @@ -71,7 +287,9 @@ public class ArbitraryResource { }) public List searchTransactions(@QueryParam("startBlock") Integer startBlock, @QueryParam("blockLimit") Integer blockLimit, @QueryParam("txGroupId") Integer txGroupId, - @QueryParam("service") Integer service, @QueryParam("address") String address, @Parameter( + @QueryParam("service") Service service, + @QueryParam("name") String name, + @QueryParam("address") String address, @Parameter( description = "whether to include confirmed, unconfirmed or both", required = true ) @QueryParam("confirmationStatus") ConfirmationStatus confirmationStatus, @Parameter( @@ -93,69 +311,15 @@ public List searchTransactions(@QueryParam("startBlock") Intege txTypes.add(TransactionType.ARBITRARY); try (final Repository repository = RepositoryManager.getRepository()) { - List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(startBlock, blockLimit, txGroupId, txTypes, - service, address, confirmationStatus, limit, offset, reverse); + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(startBlock, blockLimit, txGroupId, txTypes, + service, name, address, confirmationStatus, limit, offset, reverse); // Expand signatures to transactions - List transactions = new ArrayList(signatures.size()); + List transactions = new ArrayList<>(signatures.size()); for (byte[] signature : signatures) transactions.add(repository.getTransactionRepository().fromSignature(signature)); return transactions; - } catch (ApiException e) { - throw e; - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @GET - @Path("/raw/{signature}") - @Operation( - summary = "Fetch raw data associated with passed transaction signature", - responses = { - @ApiResponse( - description = "raw data", - content = @Content( - schema = @Schema(type = "string", format = "byte"), - mediaType = MediaType.APPLICATION_OCTET_STREAM - ) - ) - } - ) - @ApiErrors({ - ApiError.INVALID_SIGNATURE, ApiError.REPOSITORY_ISSUE, ApiError.TRANSACTION_INVALID - }) - public byte[] fetchRawData(@PathParam("signature") String signature58) { - // Decode signature - byte[] signature; - try { - signature = Base58.decode(signature58); - } catch (NumberFormatException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_SIGNATURE, e); - } - - try (final Repository repository = RepositoryManager.getRepository()) { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - - if (transactionData == null || transactionData.getType() != TransactionType.ARBITRARY) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_SIGNATURE); - - ArbitraryTransactionData arbitraryTxData = (ArbitraryTransactionData) transactionData; - - // We're really expecting to only fetch the data's hash from repository - if (arbitraryTxData.getDataType() != DataType.DATA_HASH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID); - - ArbitraryTransaction arbitraryTx = new ArbitraryTransaction(repository, arbitraryTxData); - - // For now, we only allow locally stored data - if (!arbitraryTx.isDataLocal()) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID); - - return arbitraryTx.fetchData(); - } catch (ApiException e) { - throw e; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -209,4 +373,777 @@ public String createArbitrary(ArbitraryTransactionData transactionData) { } } -} \ No newline at end of file + @GET + @Path("/relaymode") + @Operation( + summary = "Returns whether relay mode is enabled or not", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + public boolean getRelayMode(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + + return Settings.getInstance().isRelayModeEnabled(); + } + + @GET + @Path("/hosted/transactions") + @Operation( + summary = "List arbitrary transactions hosted by this node", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryTransactionData.class)) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + public List getHostedTransactions(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @Parameter(ref = "limit") @QueryParam("limit") Integer limit, + @Parameter(ref = "offset") @QueryParam("offset") Integer offset) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset); + + return hostedTransactions; + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @GET + @Path("/hosted/resources") + @Operation( + summary = "List arbitrary resources hosted by this node", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class)) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + public List getHostedResources( + @HeaderParam(Security.API_KEY_HEADER) String apiKey, + @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus, + @Parameter(ref = "limit") @QueryParam("limit") Integer limit, + @Parameter(ref = "offset") @QueryParam("offset") Integer offset, + @QueryParam("query") String query) { + + Security.checkApiCallAllowed(request); + + List resources = new ArrayList<>(); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List transactionDataList; + + if (query == null || query.equals("")) { + transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset); + } else { + transactionDataList = ArbitraryDataStorageManager.getInstance().searchHostedTransactions(repository,query, limit, offset); + } + + for (ArbitraryTransactionData transactionData : transactionDataList) { + ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); + arbitraryResourceInfo.name = transactionData.getName(); + arbitraryResourceInfo.service = transactionData.getService(); + arbitraryResourceInfo.identifier = transactionData.getIdentifier(); + if (!resources.contains(arbitraryResourceInfo)) { + resources.add(arbitraryResourceInfo); + } + } + + if (includeStatus != null && includeStatus == true) { + resources = this.addStatusToResources(resources); + } + + return resources; + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + + + @DELETE + @Path("/resource/{service}/{name}/{identifier}") + @Operation( + summary = "Delete arbitrary resource with supplied service, name and identifier", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public boolean deleteResource(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") Service service, + @PathParam("name") String name, + @PathParam("identifier") String identifier) { + + Security.checkApiCallAllowed(request); + ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); + return resource.delete(); + } + + @POST + @Path("/compute") + @Operation( + summary = "Compute nonce for raw, unsigned ARBITRARY transaction", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + description = "raw, unsigned ARBITRARY transaction in base58 encoding", + example = "raw transaction base58" + ) + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.TRANSACTION_INVALID, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String computeNonce(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String rawBytes58) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + byte[] rawBytes = Base58.decode(rawBytes58); + // We're expecting unsigned transaction, so append empty signature prior to decoding + rawBytes = Bytes.concat(rawBytes, new byte[TransactionTransformer.SIGNATURE_LENGTH]); + + TransactionData transactionData = TransactionTransformer.fromBytes(rawBytes); + if (transactionData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (transactionData.getType() != TransactionType.ARBITRARY) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + ArbitraryTransaction arbitraryTransaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); + + // Quicker validity check first before we compute nonce + ValidationResult result = arbitraryTransaction.isValid(); + if (result != ValidationResult.OK) + throw TransactionsResource.createTransactionInvalidException(request, result); + + LOGGER.info("Computing nonce..."); + arbitraryTransaction.computeNonce(); + + // Re-check, but ignores signature + result = arbitraryTransaction.isValidUnconfirmed(); + if (result != ValidationResult.OK) + throw TransactionsResource.createTransactionInvalidException(request, result); + + // Strip zeroed signature + transactionData.setSignature(null); + + byte[] bytes = ArbitraryTransactionTransformer.toBytes(transactionData); + return Base58.encode(bytes); + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + + @GET + @Path("/{service}/{name}") + @Operation( + summary = "Fetch raw data from file with supplied service, name, and relative path", + description = "An optional rebuild boolean can be supplied. If true, any existing cached data will be invalidated.", + responses = { + @ApiResponse( + description = "Path to file structure containing requested data", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public HttpServletResponse get(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") Service service, + @PathParam("name") String name, + @QueryParam("filepath") String filepath, + @QueryParam("rebuild") boolean rebuild, + @QueryParam("async") boolean async, + @QueryParam("attempts") Integer attempts) { + + // Authentication can be bypassed in the settings, for those running public QDN nodes + if (!Settings.getInstance().isQDNAuthBypassEnabled()) { + Security.checkApiCallAllowed(request); + } + + return this.download(service, name, null, filepath, rebuild, async, attempts); + } + + @GET + @Path("/{service}/{name}/{identifier}") + @Operation( + summary = "Fetch raw data from file with supplied service, name, identifier, and relative path", + description = "An optional rebuild boolean can be supplied. If true, any existing cached data will be invalidated.", + responses = { + @ApiResponse( + description = "Path to file structure containing requested data", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public HttpServletResponse get(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") Service service, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @QueryParam("filepath") String filepath, + @QueryParam("rebuild") boolean rebuild, + @QueryParam("async") boolean async, + @QueryParam("attempts") Integer attempts) { + + // Authentication can be bypassed in the settings, for those running public QDN nodes + if (!Settings.getInstance().isQDNAuthBypassEnabled()) { + Security.checkApiCallAllowed(request); + } + + return this.download(service, name, identifier, filepath, rebuild, async, attempts); + } + + + + // Upload data at supplied path + + @POST + @Path("/{service}/{name}") + @Operation( + summary = "Build raw, unsigned, ARBITRARY transaction, based on a user-supplied path", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "/Users/user/Documents/MyDirectoryOrFile" + ) + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String post(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + String path) { + Security.checkApiCallAllowed(request); + + if (path == null || path.isEmpty()) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Path not supplied"); + } + + return this.upload(Service.valueOf(serviceString), name, null, path, null, null, false); + } + + @POST + @Path("/{service}/{name}/{identifier}") + @Operation( + summary = "Build raw, unsigned, ARBITRARY transaction, based on a user-supplied path", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "/Users/user/Documents/MyDirectoryOrFile" + ) + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String post(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + String path) { + Security.checkApiCallAllowed(request); + + if (path == null || path.isEmpty()) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Path not supplied"); + } + + return this.upload(Service.valueOf(serviceString), name, identifier, path, null, null, false); + } + + + + // Upload base64-encoded data + + @POST + @Path("/{service}/{name}/base64") + @Operation( + summary = "Build raw, unsigned, ARBITRARY transaction, based on user-supplied base64 encoded data", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_OCTET_STREAM, + schema = @Schema(type = "string", format = "byte") + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String postBase64EncodedData(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + String base64) { + Security.checkApiCallAllowed(request); + + if (base64 == null) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied"); + } + + return this.upload(Service.valueOf(serviceString), name, null, null, null, base64, false); + } + + @POST + @Path("/{service}/{name}/{identifier}/base64") + @Operation( + summary = "Build raw, unsigned, ARBITRARY transaction, based on user supplied base64 encoded data", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_OCTET_STREAM, + schema = @Schema(type = "string", format = "byte") + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String postBase64EncodedData(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + String base64) { + Security.checkApiCallAllowed(request); + + if (base64 == null) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied"); + } + + return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64, false); + } + + + // Upload zipped data + + @POST + @Path("/{service}/{name}/zip") + @Operation( + summary = "Build raw, unsigned, ARBITRARY transaction, based on user-supplied zip file, encoded as base64", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_OCTET_STREAM, + schema = @Schema(type = "string", format = "byte") + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String postZippedData(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + String base64Zip) { + Security.checkApiCallAllowed(request); + + if (base64Zip == null) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied"); + } + + return this.upload(Service.valueOf(serviceString), name, null, null, null, base64Zip, true); + } + + @POST + @Path("/{service}/{name}/{identifier}/zip") + @Operation( + summary = "Build raw, unsigned, ARBITRARY transaction, based on user supplied zip file, encoded as base64", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_OCTET_STREAM, + schema = @Schema(type = "string", format = "byte") + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String postZippedData(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + String base64Zip) { + Security.checkApiCallAllowed(request); + + if (base64Zip == null) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied"); + } + + return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64Zip, true); + } + + + + // Upload plain-text data in string form + + @POST + @Path("/{service}/{name}/string") + @Operation( + summary = "Build raw, unsigned, ARBITRARY transaction, based on a user-supplied string", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "{\"title\":\"\", \"description\":\"\", \"tags\":[]}" + ) + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String postString(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + String string) { + Security.checkApiCallAllowed(request); + + if (string == null || string.isEmpty()) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data string not supplied"); + } + + return this.upload(Service.valueOf(serviceString), name, null, null, string, null, false); + } + + @POST + @Path("/{service}/{name}/{identifier}/string") + @Operation( + summary = "Build raw, unsigned, ARBITRARY transaction, based on user supplied string", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "{\"title\":\"\", \"description\":\"\", \"tags\":[]}" + ) + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String postString(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + String string) { + Security.checkApiCallAllowed(request); + + if (string == null || string.isEmpty()) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data string not supplied"); + } + + return this.upload(Service.valueOf(serviceString), name, identifier, null, string, null, false); + } + + + // Shared methods + + private String upload(Service service, String name, String identifier, String path, String string, String base64, boolean zipped) { + // Fetch public key from registered name + try (final Repository repository = RepositoryManager.getRepository()) { + NameData nameData = repository.getNameRepository().fromName(name); + if (nameData == null) { + String error = String.format("Name not registered: %s", name); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, error); + } + + if (!Controller.getInstance().isUpToDate()) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC); + } + + AccountData accountData = repository.getAccountRepository().getAccount(nameData.getOwner()); + if (accountData == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + } + byte[] publicKey = accountData.getPublicKey(); + String publicKey58 = Base58.encode(publicKey); + + if (path == null) { + // See if we have a string instead + if (string != null) { + File tempFile = File.createTempFile("qortal-", ".tmp"); + tempFile.deleteOnExit(); + BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile.toPath().toString())); + writer.write(string); + writer.newLine(); + writer.close(); + path = tempFile.toPath().toString(); + } + // ... or base64 encoded raw data + else if (base64 != null) { + File tempFile = File.createTempFile("qortal-", ".tmp"); + tempFile.deleteOnExit(); + Files.write(tempFile.toPath(), Base64.decode(base64)); + path = tempFile.toPath().toString(); + } + else { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing path or data string"); + } + } + + if (zipped) { + // Unzip the file + java.nio.file.Path tempDirectory = Files.createTempDirectory("qortal-"); + tempDirectory.toFile().deleteOnExit(); + LOGGER.info("Unzipping..."); + ZipUtils.unzip(path, tempDirectory.toString()); + path = tempDirectory.toString(); + + // Handle directories slightly differently to files + if (tempDirectory.toFile().isDirectory()) { + // The actual data will be in a randomly-named subfolder of tempDirectory + // Remove hidden folders, i.e. starting with "_", as some systems can add them, e.g. "__MACOSX" + String[] files = tempDirectory.toFile().list((parent, child) -> !child.startsWith("_")); + if (files.length == 1) { // Single directory or file only + path = Paths.get(tempDirectory.toString(), files[0]).toString(); + } + } + } + + try { + ArbitraryDataTransactionBuilder transactionBuilder = new ArbitraryDataTransactionBuilder( + repository, publicKey58, Paths.get(path), name, null, service, identifier + ); + + transactionBuilder.build(); + // Don't compute nonce - this is done by the client (or via POST /arbitrary/compute) + ArbitraryTransactionData transactionData = transactionBuilder.getArbitraryTransactionData(); + return Base58.encode(ArbitraryTransactionTransformer.toBytes(transactionData)); + + } catch (DataException | TransformationException | IllegalStateException e) { + LOGGER.info("Unable to upload data: {}", e.getMessage()); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage()); + } + + } catch (DataException | IOException e) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage()); + } + } + + private HttpServletResponse download(Service service, String name, String identifier, String filepath, boolean rebuild, boolean async, Integer maxAttempts) { + + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); + try { + + int attempts = 0; + if (maxAttempts == null) { + maxAttempts = 5; + } + + // Loop until we have data + if (async) { + // Asynchronous + arbitraryDataReader.loadAsynchronously(false, 1); + } + else { + // Synchronous + while (!Controller.isStopping()) { + attempts++; + if (!arbitraryDataReader.isBuilding()) { + try { + arbitraryDataReader.loadSynchronously(rebuild); + break; + } catch (MissingDataException e) { + if (attempts > maxAttempts) { + // Give up after 5 attempts + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later."); + } + } + } + Thread.sleep(3000L); + } + } + + java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); + if (outputPath == null) { + // Assume the resource doesn't exist + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); + } + + if (filepath == null || filepath.isEmpty()) { + // No file path supplied - so check if this is a single file resource + String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); + if (files.length == 1) { + // This is a single file resource + filepath = files[0]; + } + else { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, + "filepath is required for resources containing more than one file"); + } + } + + // TODO: limit file size that can be read into memory + java.nio.file.Path path = Paths.get(outputPath.toString(), filepath); + if (!Files.exists(path)) { + String message = String.format("No file exists at filepath: %s", filepath); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message); + } + byte[] data = Files.readAllBytes(path); + response.setContentType(context.getMimeType(path.toString())); + response.setContentLength(data.length); + response.getOutputStream().write(data); + + return response; + } catch (Exception e) { + LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); + } + } + + + private ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build) { + + // If "build=true" has been specified in the query string, build the resource before returning its status + if (build != null && build == true) { + ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null); + try { + if (!reader.isBuilding()) { + reader.loadSynchronously(false); + } + } catch (Exception e) { + // No need to handle exception, as it will be reflected in the status + } + } + + ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); + return resource.getStatus(false); + } + + private List addStatusToResources(List resources) { + // Determine and add the status of each resource + List updatedResources = new ArrayList<>(); + for (ArbitraryResourceInfo resourceInfo : resources) { + ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ResourceIdType.NAME, + resourceInfo.service, resourceInfo.identifier); + ArbitraryResourceStatus status = resource.getStatus(true); + if (status != null) { + resourceInfo.status = status; + } + updatedResources.add(resourceInfo); + } + return updatedResources; + } +} diff --git a/src/main/java/org/qortal/api/resource/AtResource.java b/src/main/java/org/qortal/api/resource/AtResource.java index 86b79da93..29a2344d5 100644 --- a/src/main/java/org/qortal/api/resource/AtResource.java +++ b/src/main/java/org/qortal/api/resource/AtResource.java @@ -25,7 +25,6 @@ import org.qortal.api.ApiErrors; import org.qortal.api.ApiException; import org.qortal.api.ApiExceptionFactory; -import org.qortal.at.QortalAtLoggerFactory; import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.data.transaction.DeployAtTransactionData; @@ -147,8 +146,7 @@ public byte[] getDataByAddress(@PathParam("ataddress") String atAddress) { ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); byte[] stateData = atStateData.getStateData(); - QortalAtLoggerFactory loggerFactory = QortalAtLoggerFactory.getInstance(); - byte[] dataBytes = MachineState.extractDataBytes(loggerFactory, stateData); + byte[] dataBytes = MachineState.extractDataBytes(stateData); return dataBytes; } catch (ApiException e) { diff --git a/src/main/java/org/qortal/api/resource/BlocksResource.java b/src/main/java/org/qortal/api/resource/BlocksResource.java index 6686268ed..28d54a448 100644 --- a/src/main/java/org/qortal/api/resource/BlocksResource.java +++ b/src/main/java/org/qortal/api/resource/BlocksResource.java @@ -1,5 +1,6 @@ package org.qortal.api.resource; +import com.google.common.primitives.Ints; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.ArraySchema; @@ -8,7 +9,14 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.List; import javax.servlet.http.HttpServletRequest; @@ -20,18 +28,25 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import org.qortal.account.Account; import org.qortal.api.ApiError; import org.qortal.api.ApiErrors; import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.model.BlockMintingInfo; import org.qortal.api.model.BlockSignerSummary; +import org.qortal.block.Block; +import org.qortal.controller.Controller; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountData; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.BlockArchiveReader; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformer; import org.qortal.utils.Base58; @Path("/blocks") @@ -60,7 +75,8 @@ public class BlocksResource { @ApiErrors({ ApiError.INVALID_SIGNATURE, ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE }) - public BlockData getBlock(@PathParam("signature") String signature58) { + public BlockData getBlock(@PathParam("signature") String signature58, + @QueryParam("includeOnlineSignatures") Boolean includeOnlineSignatures) { // Decode signature byte[] signature; try { @@ -70,16 +86,80 @@ public BlockData getBlock(@PathParam("signature") String signature58) { } try (final Repository repository = RepositoryManager.getRepository()) { + // Check the database first BlockData blockData = repository.getBlockRepository().fromSignature(signature); - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + if (blockData != null) { + if (includeOnlineSignatures == null || includeOnlineSignatures == false) { + blockData.setOnlineAccountsSignatures(null); + } + return blockData; + } - return blockData; + // Not found, so try the block archive + blockData = repository.getBlockArchiveRepository().fromSignature(signature); + if (blockData != null) { + if (includeOnlineSignatures == null || includeOnlineSignatures == false) { + blockData.setOnlineAccountsSignatures(null); + } + return blockData; + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } } + @GET + @Path("/signature/{signature}/data") + @Operation( + summary = "Fetch serialized, base58 encoded block data using base58 signature", + description = "Returns serialized data for the block that matches the given signature", + responses = { + @ApiResponse( + description = "the block data", + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ + ApiError.INVALID_SIGNATURE, ApiError.BLOCK_UNKNOWN, ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE + }) + public String getSerializedBlockData(@PathParam("signature") String signature58) { + // Decode signature + byte[] signature; + try { + signature = Base58.decode(signature58); + } catch (NumberFormatException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_SIGNATURE, e); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + + // Check the database first + BlockData blockData = repository.getBlockRepository().fromSignature(signature); + if (blockData != null) { + Block block = new Block(repository, blockData); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); + bytes.write(BlockTransformer.toBytes(block)); + return Base58.encode(bytes.toByteArray()); + } + + // Not found, so try the block archive + byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, false, repository); + if (bytes != null) { + return Base58.encode(bytes); + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); + } catch (DataException | IOException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + @GET @Path("/signature/{signature}/transactions") @Operation( @@ -117,8 +197,12 @@ public List getBlockTransactions(@PathParam("signature") String } try (final Repository repository = RepositoryManager.getRepository()) { - if (repository.getBlockRepository().getHeightFromSignature(signature) == 0) + // Check if the block exists in either the database or archive + if (repository.getBlockRepository().getHeightFromSignature(signature) == 0 && + repository.getBlockArchiveRepository().getHeightFromSignature(signature) == 0) { + // Not found in either the database or archive throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } return repository.getBlockRepository().getTransactionsFromSignature(signature, limit, offset, reverse); } catch (DataException e) { @@ -147,7 +231,19 @@ public List getBlockTransactions(@PathParam("signature") String }) public BlockData getFirstBlock() { try (final Repository repository = RepositoryManager.getRepository()) { - return repository.getBlockRepository().fromHeight(1); + // Check the database first + BlockData blockData = repository.getBlockRepository().fromHeight(1); + if (blockData != null) { + return blockData; + } + + // Try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(1); + if (blockData != null) { + return blockData; + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -172,9 +268,15 @@ public BlockData getFirstBlock() { @ApiErrors({ ApiError.REPOSITORY_ISSUE }) - public BlockData getLastBlock() { + public BlockData getLastBlock(@QueryParam("includeOnlineSignatures") Boolean includeOnlineSignatures) { try (final Repository repository = RepositoryManager.getRepository()) { - return repository.getBlockRepository().getLastBlock(); + BlockData blockData = repository.getBlockRepository().getLastBlock(); + + if (includeOnlineSignatures == null || includeOnlineSignatures == false) { + blockData.setOnlineAccountsSignatures(null); + } + + return blockData; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -209,17 +311,28 @@ public BlockData getChild(@PathParam("signature") String signature58) { } try (final Repository repository = RepositoryManager.getRepository()) { - BlockData blockData = repository.getBlockRepository().fromSignature(signature); + BlockData childBlockData = null; - // Check block exists - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + // Check if block exists in database + BlockData blockData = repository.getBlockRepository().fromSignature(signature); + if (blockData != null) { + return repository.getBlockRepository().fromReference(signature); + } - BlockData childBlockData = repository.getBlockRepository().fromReference(signature); + // Not found, so try the archive + // This also checks that the parent block exists + // It will return null if either the parent or child don't exit + childBlockData = repository.getBlockArchiveRepository().fromReference(signature); // Check child block exists - if (childBlockData == null) + if (childBlockData == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + + // Check child block's reference matches the supplied signature + if (!Arrays.equals(childBlockData.getReference(), signature)) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } return childBlockData; } catch (DataException e) { @@ -285,13 +398,20 @@ public int getHeight(@PathParam("signature") String signature58) { } try (final Repository repository = RepositoryManager.getRepository()) { + // Firstly check the database BlockData blockData = repository.getBlockRepository().fromSignature(signature); + if (blockData != null) { + return blockData.getHeight(); + } - // Check block exists - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + // Not found, so try the archive + blockData = repository.getBlockArchiveRepository().fromSignature(signature); + if (blockData != null) { + return blockData.getHeight(); + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); - return blockData.getHeight(); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -316,13 +436,101 @@ public int getHeight(@PathParam("signature") String signature58) { @ApiErrors({ ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE }) - public BlockData getByHeight(@PathParam("height") int height) { + public BlockData getByHeight(@PathParam("height") int height, + @QueryParam("includeOnlineSignatures") Boolean includeOnlineSignatures) { try (final Repository repository = RepositoryManager.getRepository()) { + // Firstly check the database BlockData blockData = repository.getBlockRepository().fromHeight(height); - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + if (blockData != null) { + if (includeOnlineSignatures == null || includeOnlineSignatures == false) { + blockData.setOnlineAccountsSignatures(null); + } + return blockData; + } + + // Not found, so try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(height); + if (blockData != null) { + if (includeOnlineSignatures == null || includeOnlineSignatures == false) { + blockData.setOnlineAccountsSignatures(null); + } + return blockData; + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); - return blockData; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @GET + @Path("/byheight/{height}/mintinginfo") + @Operation( + summary = "Fetch block minter info using block height", + description = "Returns the minter info for the block with given height", + responses = { + @ApiResponse( + description = "the block", + content = @Content( + schema = @Schema( + implementation = BlockData.class + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE + }) + public BlockMintingInfo getBlockMintingInfoByHeight(@PathParam("height") int height) { + try (final Repository repository = RepositoryManager.getRepository()) { + // Try the database + BlockData blockData = repository.getBlockRepository().fromHeight(height); + if (blockData == null) { + + // Not found, so try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(height); + if (blockData == null) { + + // Still not found + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + } + + Block block = new Block(repository, blockData); + BlockData parentBlockData = repository.getBlockRepository().fromSignature(blockData.getReference()); + if (parentBlockData == null) { + // Parent block not found - try the archive + parentBlockData = repository.getBlockArchiveRepository().fromSignature(blockData.getReference()); + if (parentBlockData == null) { + + // Still not found + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + } + + int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey()); + if (minterLevel == 0) + // This may be unavailable when requesting a trimmed block + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + BigInteger distance = block.calcKeyDistance(parentBlockData.getHeight(), parentBlockData.getSignature(), blockData.getMinterPublicKey(), minterLevel); + double ratio = new BigDecimal(distance).divide(new BigDecimal(block.MAX_DISTANCE), 40, RoundingMode.DOWN).doubleValue(); + long timestamp = block.calcTimestamp(parentBlockData, blockData.getMinterPublicKey(), minterLevel); + long timeDelta = timestamp - parentBlockData.getTimestamp(); + + BlockMintingInfo blockMintingInfo = new BlockMintingInfo(); + blockMintingInfo.minterPublicKey = blockData.getMinterPublicKey(); + blockMintingInfo.minterLevel = minterLevel; + blockMintingInfo.onlineAccountsCount = blockData.getOnlineAccountsCount(); + blockMintingInfo.maxDistance = new BigDecimal(block.MAX_DISTANCE); + blockMintingInfo.keyDistance = distance; + blockMintingInfo.keyDistanceRatio = ratio; + blockMintingInfo.timestamp = timestamp; + blockMintingInfo.timeDelta = timeDelta; + + return blockMintingInfo; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -346,15 +554,37 @@ public BlockData getByHeight(@PathParam("height") int height) { @ApiErrors({ ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE }) - public BlockData getByTimestamp(@PathParam("timestamp") long timestamp) { + public BlockData getByTimestamp(@PathParam("timestamp") long timestamp, + @QueryParam("includeOnlineSignatures") Boolean includeOnlineSignatures) { try (final Repository repository = RepositoryManager.getRepository()) { + BlockData blockData = null; + + // Try the Blocks table int height = repository.getBlockRepository().getHeightFromTimestamp(timestamp); - if (height == 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + if (height > 1) { + // Found match in Blocks table + blockData = repository.getBlockRepository().fromHeight(height); + if (includeOnlineSignatures == null || includeOnlineSignatures == false) { + blockData.setOnlineAccountsSignatures(null); + } + return blockData; + } - BlockData blockData = repository.getBlockRepository().fromHeight(height); - if (blockData == null) + // Not found in Blocks table, so try the archive + height = repository.getBlockArchiveRepository().getHeightFromTimestamp(timestamp); + if (height > 1) { + // Found match in archive + blockData = repository.getBlockArchiveRepository().fromHeight(height); + } + + // Ensure block exists + if (blockData == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + + if (includeOnlineSignatures == null || includeOnlineSignatures == false) { + blockData.setOnlineAccountsSignatures(null); + } return blockData; } catch (DataException e) { @@ -391,9 +621,14 @@ public List getBlockRange(@PathParam("height") int height, @Parameter for (/* count already set */; count > 0; --count, ++height) { BlockData blockData = repository.getBlockRepository().fromHeight(height); - if (blockData == null) - // Run out of blocks! - break; + if (blockData == null) { + // Not found - try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(height); + if (blockData == null) { + // Run out of blocks! + break; + } + } blocks.add(blockData); } @@ -438,7 +673,29 @@ public List getBlockSummariesBySigner(@PathParam("address") St if (accountData == null || accountData.getPublicKey() == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.PUBLIC_KEY_NOT_FOUND); - return repository.getBlockRepository().getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse); + + List summaries = repository.getBlockRepository() + .getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse); + + // Add any from the archive + List archivedSummaries = repository.getBlockArchiveRepository() + .getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse); + if (archivedSummaries != null && !archivedSummaries.isEmpty()) { + summaries.addAll(archivedSummaries); + } + else { + summaries = archivedSummaries; + } + + // Sort the results (because they may have been obtained from two places) + if (reverse != null && reverse) { + summaries.sort((s1, s2) -> Integer.valueOf(s2.getHeight()).compareTo(Integer.valueOf(s1.getHeight()))); + } + else { + summaries.sort(Comparator.comparing(s -> Integer.valueOf(s.getHeight()))); + } + + return summaries; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -474,7 +731,117 @@ public List getBlockSigners(@QueryParam("address") List getBlockSummaries( + @QueryParam("start") Integer startHeight, + @QueryParam("end") Integer endHeight, + @Parameter(ref = "count") @QueryParam("count") Integer count) { + // Check up to 2 out of 3 params + if (startHeight != null && endHeight != null && count != null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Check values + if ((startHeight != null && startHeight < 1) || (endHeight != null && endHeight < 1) || (count != null && count < 1)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + try (final Repository repository = RepositoryManager.getRepository()) { + + /* + * start end count result + * 10 40 null blocks 10 to 39 (excludes end block, ignore count) + * + * null null null blocks 1 to 50 (assume count=50, maybe start=1) + * 30 null null blocks 30 to 79 (assume count=50) + * 30 null 10 blocks 30 to 39 + * + * null null 50 last 50 blocks? so if max(blocks.height) is 200, then blocks 151 to 200 + * null 200 null blocks 150 to 199 (excludes end block, assume count=50) + * null 200 10 blocks 190 to 199 (excludes end block) + */ + + List blockSummaries = new ArrayList<>(); + + // Use the latest X blocks if only a count is specified + if (startHeight == null && endHeight == null && count != null) { + BlockData chainTip = repository.getBlockRepository().getLastBlock(); + startHeight = chainTip.getHeight() - count; + endHeight = chainTip.getHeight(); + } + + // ... otherwise default the start height to 1 + if (startHeight == null && endHeight == null) { + startHeight = 1; + } + + // Default the count to 50 + if (count == null) { + count = 50; + } + + // If both a start and end height exist, ignore the count + if (startHeight != null && endHeight != null) { + if (startHeight > 0 && endHeight > 0) { + count = Integer.MAX_VALUE; + } + } + + // Derive start height from end height if missing + if (startHeight == null || startHeight == 0) { + if (endHeight != null && endHeight > 0) { + if (count != null) { + startHeight = endHeight - count; + } + } + } + + for (/* count already set */; count > 0; --count, ++startHeight) { + if (endHeight != null && startHeight >= endHeight) { + break; + } + BlockData blockData = repository.getBlockRepository().fromHeight(startHeight); + if (blockData == null) { + // Not found - try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(startHeight); + if (blockData == null) { + // Run out of blocks! + break; + } + } + + if (blockData != null) { + BlockSummaryData blockSummaryData = new BlockSummaryData(blockData); + blockSummaries.add(blockSummaryData); + } + } + + return blockSummaries; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } diff --git a/src/main/java/org/qortal/api/resource/BootstrapResource.java b/src/main/java/org/qortal/api/resource/BootstrapResource.java new file mode 100644 index 000000000..b9382dcbe --- /dev/null +++ b/src/main/java/org/qortal/api/resource/BootstrapResource.java @@ -0,0 +1,95 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.api.ApiError; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.repository.Bootstrap; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import java.io.IOException; + + +@Path("/bootstrap") +@Tag(name = "Bootstrap") +public class BootstrapResource { + + private static final Logger LOGGER = LogManager.getLogger(BootstrapResource.class); + + @Context + HttpServletRequest request; + + @POST + @Path("/create") + @Operation( + summary = "Create bootstrap", + description = "Builds a bootstrap file for distribution", + responses = { + @ApiResponse( + description = "path to file on success, an exception on failure", + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String createBootstrap(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + + Bootstrap bootstrap = new Bootstrap(repository); + try { + bootstrap.checkRepositoryState(); + } catch (DataException e) { + LOGGER.info("Not ready to create bootstrap: {}", e.getMessage()); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage()); + } + bootstrap.validateBlockchain(); + return bootstrap.create(); + + } catch (DataException | InterruptedException | IOException e) { + LOGGER.info("Unable to create bootstrap", e); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage()); + } + } + + @GET + @Path("/validate") + @Operation( + summary = "Validate blockchain", + description = "Useful to check database integrity prior to creating or after installing a bootstrap. " + + "This process is intensive and can take over an hour to run.", + responses = { + @ApiResponse( + description = "true if valid, false if invalid", + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public boolean validateBootstrap(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + + Bootstrap bootstrap = new Bootstrap(repository); + return bootstrap.validateCompleteBlockchain(); + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); + } + } +} diff --git a/src/main/java/org/qortal/api/resource/ChatResource.java b/src/main/java/org/qortal/api/resource/ChatResource.java index 9a8fc8d58..be8bd7d7a 100644 --- a/src/main/java/org/qortal/api/resource/ChatResource.java +++ b/src/main/java/org/qortal/api/resource/ChatResource.java @@ -7,16 +7,13 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; import java.util.List; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.QueryParam; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; @@ -156,7 +153,8 @@ public ActiveChats getActiveChats(@PathParam("address") String address) { } ) @ApiErrors({ApiError.TRANSACTION_INVALID, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE}) - public String buildChat(ChatTransactionData transactionData) { + @SecurityRequirement(name = "apiKey") + public String buildChat(@HeaderParam(Security.API_KEY_HEADER) String apiKey, ChatTransactionData transactionData) { Security.checkApiCallAllowed(request); try (final Repository repository = RepositoryManager.getRepository()) { @@ -203,7 +201,8 @@ public String buildChat(ChatTransactionData transactionData) { } ) @ApiErrors({ApiError.TRANSACTION_INVALID, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE}) - public String buildChat(String rawBytes58) { + @SecurityRequirement(name = "apiKey") + public String buildChat(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String rawBytes58) { Security.checkApiCallAllowed(request); try (final Repository repository = RepositoryManager.getRepository()) { diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java new file mode 100644 index 000000000..6cfa130a1 --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java @@ -0,0 +1,368 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; + +import java.util.Arrays; +import java.util.Random; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.HeaderParam; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.CrossChainBuildRequest; +import org.qortal.api.model.CrossChainDualSecretRequest; +import org.qortal.api.model.CrossChainTradeRequest; +import org.qortal.asset.Asset; +import org.qortal.crosschain.BitcoinACCTv1; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.AcctMode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.Transformer; +import org.qortal.transform.transaction.DeployAtTransactionTransformer; +import org.qortal.transform.transaction.MessageTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +@Path("/crosschain/BitcoinACCTv1") +@Tag(name = "Cross-Chain (BitcoinACCTv1)") +public class CrossChainBitcoinACCTv1Resource { + + @Context + HttpServletRequest request; + + @POST + @Path("/build") + @Operation( + summary = "Build Bitcoin cross-chain trading AT", + description = "Returns raw, unsigned DEPLOY_AT transaction", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = CrossChainBuildRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_DATA, ApiError.INVALID_REFERENCE, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String buildTrade(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainBuildRequest tradeRequest) { + Security.checkApiCallAllowed(request); + + byte[] creatorPublicKey = tradeRequest.creatorPublicKey; + + if (creatorPublicKey == null || creatorPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + + if (tradeRequest.hashOfSecretB == null || tradeRequest.hashOfSecretB.length != Bitcoiny.HASH160_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (tradeRequest.tradeTimeout == null) + tradeRequest.tradeTimeout = 7 * 24 * 60; // 7 days + else + if (tradeRequest.tradeTimeout < 10 || tradeRequest.tradeTimeout > 50000) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (tradeRequest.qortAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (tradeRequest.fundingQortAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + // funding amount must exceed initial + final + if (tradeRequest.fundingQortAmount <= tradeRequest.qortAmount) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (tradeRequest.bitcoinAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + try (final Repository repository = RepositoryManager.getRepository()) { + PublicKeyAccount creatorAccount = new PublicKeyAccount(repository, creatorPublicKey); + + byte[] creationBytes = BitcoinACCTv1.buildQortalAT(creatorAccount.getAddress(), tradeRequest.bitcoinPublicKeyHash, tradeRequest.hashOfSecretB, + tradeRequest.qortAmount, tradeRequest.bitcoinAmount, tradeRequest.tradeTimeout); + + long txTimestamp = NTP.getTime(); + byte[] lastReference = creatorAccount.getLastReference(); + if (lastReference == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_REFERENCE); + + long fee = 0; + String name = "QORT-BTC cross-chain trade"; + String description = "Qortal-Bitcoin cross-chain trade"; + String atType = "ACCT"; + String tags = "QORT-BTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, creatorAccount.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, tradeRequest.fundingQortAmount, Asset.QORT); + + Transaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + ValidationResult result = deployAtTransaction.isValidUnconfirmed(); + if (result != ValidationResult.OK) + throw TransactionsResource.createTransactionInvalidException(request, result); + + byte[] bytes = DeployAtTransactionTransformer.toBytes(deployAtTransactionData); + return Base58.encode(bytes); + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST + @Path("/trademessage") + @Operation( + summary = "Builds raw, unsigned 'trade' MESSAGE transaction that sends cross-chain trade recipient address, triggering 'trade' mode", + description = "Specify address of cross-chain AT that needs to be messaged, and signature of 'offer' MESSAGE from trade partner.
" + + "AT needs to be in 'offer' mode. Messages sent to an AT in any other mode will be ignored, but still cost fees to send!
" + + "You need to sign output with trade private key otherwise the MESSAGE transaction will be invalid.", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = CrossChainTradeRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String buildTradeMessage(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainTradeRequest tradeRequest) { + Security.checkApiCallAllowed(request); + + byte[] tradePublicKey = tradeRequest.tradePublicKey; + + if (tradePublicKey == null || tradePublicKey.length != Transformer.PUBLIC_KEY_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + + if (tradeRequest.atAddress == null || !Crypto.isValidAtAddress(tradeRequest.atAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (tradeRequest.messageTransactionSignature == null || !Crypto.isValidAddress(tradeRequest.messageTransactionSignature)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = fetchAtDataWithChecking(repository, tradeRequest.atAddress); + CrossChainTradeData crossChainTradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + + if (crossChainTradeData.mode != AcctMode.OFFERING) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Does supplied public key match trade public key? + if (!Crypto.toAddress(tradePublicKey).equals(crossChainTradeData.qortalCreatorTradeAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + + TransactionData transactionData = repository.getTransactionRepository().fromSignature(tradeRequest.messageTransactionSignature); + if (transactionData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_UNKNOWN); + + if (transactionData.getType() != TransactionType.MESSAGE) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID); + + MessageTransactionData messageTransactionData = (MessageTransactionData) transactionData; + byte[] messageData = messageTransactionData.getData(); + BitcoinACCTv1.OfferMessageData offerMessageData = BitcoinACCTv1.extractOfferMessageData(messageData); + if (offerMessageData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID); + + // Good to make MESSAGE + + byte[] aliceForeignPublicKeyHash = offerMessageData.partnerBitcoinPKH; + byte[] hashOfSecretA = offerMessageData.hashOfSecretA; + int lockTimeA = (int) offerMessageData.lockTimeA; + + String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(messageTransactionData.getTimestamp(), lockTimeA); + + byte[] outgoingMessageData = BitcoinACCTv1.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + byte[] messageTransactionBytes = buildAtMessage(repository, tradePublicKey, tradeRequest.atAddress, outgoingMessageData); + + return Base58.encode(messageTransactionBytes); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST + @Path("/redeemmessage") + @Operation( + summary = "Builds raw, unsigned 'redeem' MESSAGE transaction that sends secrets to AT, releasing funds to partner", + description = "Specify address of cross-chain AT that needs to be messaged, both 32-byte secrets and an address for receiving QORT from AT.
" + + "AT needs to be in 'trade' mode. Messages sent to an AT in any other mode will be ignored, but still cost fees to send!
" + + "You need to sign output with account the AT considers the trade 'partner' otherwise the MESSAGE transaction will be invalid.", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = CrossChainDualSecretRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String buildRedeemMessage(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainDualSecretRequest secretRequest) { + Security.checkApiCallAllowed(request); + + byte[] partnerPublicKey = secretRequest.partnerPublicKey; + + if (partnerPublicKey == null || partnerPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + + if (secretRequest.atAddress == null || !Crypto.isValidAtAddress(secretRequest.atAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (secretRequest.secretA == null || secretRequest.secretA.length != BitcoinACCTv1.SECRET_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (secretRequest.secretB == null || secretRequest.secretB.length != BitcoinACCTv1.SECRET_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (secretRequest.receivingAddress == null || !Crypto.isValidAddress(secretRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = fetchAtDataWithChecking(repository, secretRequest.atAddress); + CrossChainTradeData crossChainTradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + + if (crossChainTradeData.mode != AcctMode.TRADING) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + String partnerAddress = Crypto.toAddress(partnerPublicKey); + + // MESSAGE must come from address that AT considers trade partner + if (!crossChainTradeData.qortalPartnerAddress.equals(partnerAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + // Good to make MESSAGE + + byte[] messageData = BitcoinACCTv1.buildRedeemMessage(secretRequest.secretA, secretRequest.secretB, secretRequest.receivingAddress); + byte[] messageTransactionBytes = buildAtMessage(repository, partnerPublicKey, secretRequest.atAddress, messageData); + + return Base58.encode(messageTransactionBytes); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + // Must be correct AT - check functionality using code hash + if (!Arrays.equals(atData.getCodeHash(), BitcoinACCTv1.CODE_BYTES_HASH)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // No point sending message to AT that's finished + if (atData.getIsFinished()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + return atData; + } + + private byte[] buildAtMessage(Repository repository, byte[] senderPublicKey, String atAddress, byte[] messageData) throws DataException { + long txTimestamp = NTP.getTime(); + + // senderPublicKey could be ephemeral trade public key where there is no corresponding account and hence no reference + String senderAddress = Crypto.toAddress(senderPublicKey); + byte[] lastReference = repository.getAccountRepository().getLastReference(senderAddress); + final boolean requiresPoW = lastReference == null; + + if (requiresPoW) { + Random random = new Random(); + lastReference = new byte[Transformer.SIGNATURE_LENGTH]; + random.nextBytes(lastReference); + } + + int version = 4; + int nonce = 0; + long amount = 0L; + Long assetId = null; // no assetId as amount is zero + Long fee = 0L; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, senderPublicKey, fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, atAddress, amount, assetId, messageData, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + if (requiresPoW) { + messageTransaction.computeNonce(); + } else { + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + } + + ValidationResult result = messageTransaction.isValidUnconfirmed(); + if (result != ValidationResult.OK) + throw TransactionsResource.createTransactionInvalidException(request, result); + + try { + return MessageTransactionTransformer.toBytes(messageTransactionData); + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + } + } + +} diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java new file mode 100644 index 000000000..834c7b819 --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java @@ -0,0 +1,177 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; + +import java.util.List; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.HeaderParam; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + +import org.bitcoinj.core.Transaction; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.crosschain.BitcoinSendRequest; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.SimpleTransaction; + +@Path("/crosschain/btc") +@Tag(name = "Cross-Chain (Bitcoin)") +public class CrossChainBitcoinResource { + + @Context + HttpServletRequest request; + + @POST + @Path("/walletbalance") + @Operation( + summary = "Returns BTC balance for hierarchical, deterministic BIP32 wallet", + description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + description = "BIP32 'm' private/public key in base58", + example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "balance (satoshis)")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String getBitcoinWalletBalance(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) { + Security.checkApiCallAllowed(request); + + Bitcoin bitcoin = Bitcoin.getInstance(); + + if (!bitcoin.isValidDeterministicKey(key58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + try { + Long balance = bitcoin.getWalletBalanceFromTransactions(key58); + if (balance == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + + return balance.toString(); + + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + } + + @POST + @Path("/wallettransactions") + @Operation( + summary = "Returns transactions for hierarchical, deterministic BIP32 wallet", + description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + description = "BIP32 'm' private/public key in base58", + example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(array = @ArraySchema( schema = @Schema( implementation = SimpleTransaction.class ) ) ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + @SecurityRequirement(name = "apiKey") + public List getBitcoinWalletTransactions(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) { + Security.checkApiCallAllowed(request); + + Bitcoin bitcoin = Bitcoin.getInstance(); + + if (!bitcoin.isValidDeterministicKey(key58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + try { + return bitcoin.getWalletTransactions(key58); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + } + + @POST + @Path("/send") + @Operation( + summary = "Sends BTC from hierarchical, deterministic BIP32 wallet to specific address", + description = "Currently only supports 'legacy' P2PKH Bitcoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = BitcoinSendRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "transaction hash")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String sendBitcoin(@HeaderParam(Security.API_KEY_HEADER) String apiKey, BitcoinSendRequest bitcoinSendRequest) { + Security.checkApiCallAllowed(request); + + if (bitcoinSendRequest.bitcoinAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + if (bitcoinSendRequest.feePerByte != null && bitcoinSendRequest.feePerByte <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + Bitcoin bitcoin = Bitcoin.getInstance(); + + if (!bitcoin.isValidAddress(bitcoinSendRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (!bitcoin.isValidDeterministicKey(bitcoinSendRequest.xprv58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + Transaction spendTransaction = bitcoin.buildSpend(bitcoinSendRequest.xprv58, + bitcoinSendRequest.receivingAddress, + bitcoinSendRequest.bitcoinAmount, + bitcoinSendRequest.feePerByte); + + if (spendTransaction == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE); + + try { + bitcoin.broadcastTransaction(spendTransaction); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + + return spendTransaction.getTxId().toString(); + } + +} diff --git a/src/main/java/org/qortal/api/resource/CrossChainDogecoinACCTv1Resource.java b/src/main/java/org/qortal/api/resource/CrossChainDogecoinACCTv1Resource.java new file mode 100644 index 000000000..c00874b4c --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainDogecoinACCTv1Resource.java @@ -0,0 +1,143 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.CrossChainSecretRequest; +import org.qortal.crosschain.AcctMode; +import org.qortal.crosschain.DogecoinACCTv1; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.Transformer; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.HeaderParam; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import java.util.Arrays; + +@Path("/crosschain/DogecoinACCTv1") +@Tag(name = "Cross-Chain (DogecoinACCTv1)") +public class CrossChainDogecoinACCTv1Resource { + + @Context + HttpServletRequest request; + + @POST + @Path("/redeemmessage") + @Operation( + summary = "Signs and broadcasts a 'redeem' MESSAGE transaction that sends secrets to AT, releasing funds to partner", + description = "Specify address of cross-chain AT that needs to be messaged, Alice's trade private key, the 32-byte secret,
" + + "and an address for receiving QORT from AT. All of these can be found in Alice's trade bot data.
" + + "AT needs to be in 'trade' mode. Messages sent to an AT in any other mode will be ignored, but still cost fees to send!
" + + "You need to use the private key that the AT considers the trade 'partner' otherwise the MESSAGE transaction will be invalid.", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = CrossChainSecretRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public boolean buildRedeemMessage(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainSecretRequest secretRequest) { + Security.checkApiCallAllowed(request); + + byte[] partnerPrivateKey = secretRequest.partnerPrivateKey; + + if (partnerPrivateKey == null || partnerPrivateKey.length != Transformer.PRIVATE_KEY_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + if (secretRequest.atAddress == null || !Crypto.isValidAtAddress(secretRequest.atAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (secretRequest.secret == null || secretRequest.secret.length != DogecoinACCTv1.SECRET_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (secretRequest.receivingAddress == null || !Crypto.isValidAddress(secretRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = fetchAtDataWithChecking(repository, secretRequest.atAddress); + CrossChainTradeData crossChainTradeData = DogecoinACCTv1.getInstance().populateTradeData(repository, atData); + + if (crossChainTradeData.mode != AcctMode.TRADING) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + byte[] partnerPublicKey = new PrivateKeyAccount(null, partnerPrivateKey).getPublicKey(); + String partnerAddress = Crypto.toAddress(partnerPublicKey); + + // MESSAGE must come from address that AT considers trade partner + if (!crossChainTradeData.qortalPartnerAddress.equals(partnerAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + // Good to make MESSAGE + + byte[] messageData = DogecoinACCTv1.buildRedeemMessage(secretRequest.secret, secretRequest.receivingAddress); + + PrivateKeyAccount sender = new PrivateKeyAccount(repository, partnerPrivateKey); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, secretRequest.atAddress, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID); + + return true; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + // Must be correct AT - check functionality using code hash + if (!Arrays.equals(atData.getCodeHash(), DogecoinACCTv1.CODE_BYTES_HASH)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // No point sending message to AT that's finished + if (atData.getIsFinished()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + return atData; + } + +} diff --git a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java new file mode 100644 index 000000000..189a53d30 --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java @@ -0,0 +1,175 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.bitcoinj.core.Transaction; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.crosschain.DogecoinSendRequest; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Dogecoin; +import org.qortal.crosschain.SimpleTransaction; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.HeaderParam; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import java.util.List; + +@Path("/crosschain/doge") +@Tag(name = "Cross-Chain (Dogecoin)") +public class CrossChainDogecoinResource { + + @Context + HttpServletRequest request; + + @POST + @Path("/walletbalance") + @Operation( + summary = "Returns DOGE balance for hierarchical, deterministic BIP32 wallet", + description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + description = "BIP32 'm' private/public key in base58", + example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "balance (satoshis)")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String getDogecoinWalletBalance(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) { + Security.checkApiCallAllowed(request); + + Dogecoin dogecoin = Dogecoin.getInstance(); + + if (!dogecoin.isValidDeterministicKey(key58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + try { + Long balance = dogecoin.getWalletBalanceFromTransactions(key58); + if (balance == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + + return balance.toString(); + + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + } + + @POST + @Path("/wallettransactions") + @Operation( + summary = "Returns transactions for hierarchical, deterministic BIP32 wallet", + description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + description = "BIP32 'm' private/public key in base58", + example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(array = @ArraySchema( schema = @Schema( implementation = SimpleTransaction.class ) ) ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + @SecurityRequirement(name = "apiKey") + public List getDogecoinWalletTransactions(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) { + Security.checkApiCallAllowed(request); + + Dogecoin dogecoin = Dogecoin.getInstance(); + + if (!dogecoin.isValidDeterministicKey(key58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + try { + return dogecoin.getWalletTransactions(key58); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + } + + @POST + @Path("/send") + @Operation( + summary = "Sends DOGE from hierarchical, deterministic BIP32 wallet to specific address", + description = "Currently only supports 'legacy' P2PKH Dogecoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = DogecoinSendRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "transaction hash")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String sendBitcoin(@HeaderParam(Security.API_KEY_HEADER) String apiKey, DogecoinSendRequest dogecoinSendRequest) { + Security.checkApiCallAllowed(request); + + if (dogecoinSendRequest.dogecoinAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + if (dogecoinSendRequest.feePerByte != null && dogecoinSendRequest.feePerByte <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + Dogecoin dogecoin = Dogecoin.getInstance(); + + if (!dogecoin.isValidAddress(dogecoinSendRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (!dogecoin.isValidDeterministicKey(dogecoinSendRequest.xprv58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + Transaction spendTransaction = dogecoin.buildSpend(dogecoinSendRequest.xprv58, + dogecoinSendRequest.receivingAddress, + dogecoinSendRequest.dogecoinAmount, + dogecoinSendRequest.feePerByte); + + if (spendTransaction == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE); + + try { + dogecoin.broadcastTransaction(spendTransaction); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + + return spendTransaction.getTxId().toString(); + } + +} diff --git a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java new file mode 100644 index 000000000..fbcde1a68 --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java @@ -0,0 +1,657 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; + +import java.math.BigDecimal; +import java.util.List; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.*; +import org.bitcoinj.script.Script; +import org.qortal.api.*; +import org.qortal.api.model.CrossChainBitcoinyHTLCStatus; +import org.qortal.crosschain.*; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +@Path("/crosschain/htlc") +@Tag(name = "Cross-Chain (Hash time-locked contracts)") +public class CrossChainHtlcResource { + + private static final Logger LOGGER = LogManager.getLogger(CrossChainHtlcResource.class); + + @Context + HttpServletRequest request; + + @GET + @Path("/address/{blockchain}/{refundPKH}/{locktime}/{redeemPKH}/{hashOfSecret}") + @Operation( + summary = "Returns HTLC address based on trade info", + description = "Public key hashes (PKH) and hash of secret should be 20 bytes (base58 encoded). Locktime is seconds since epoch.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_CRITERIA}) + public String deriveHtlcAddress(@PathParam("blockchain") String blockchainName, + @PathParam("refundPKH") String refundPKH, + @PathParam("locktime") int lockTime, + @PathParam("redeemPKH") String redeemPKH, + @PathParam("hashOfSecret") String hashOfSecret) { + SupportedBlockchain blockchain = SupportedBlockchain.valueOf(blockchainName); + if (blockchain == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + byte[] refunderPubKeyHash; + byte[] redeemerPubKeyHash; + byte[] decodedHashOfSecret; + + try { + refunderPubKeyHash = Base58.decode(refundPKH); + redeemerPubKeyHash = Base58.decode(redeemPKH); + + if (refunderPubKeyHash.length != 20 || redeemerPubKeyHash.length != 20) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + } catch (IllegalArgumentException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + } + + try { + decodedHashOfSecret = Base58.decode(hashOfSecret); + if (decodedHashOfSecret.length != 20) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } catch (IllegalArgumentException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + byte[] redeemScript = BitcoinyHTLC.buildScript(refunderPubKeyHash, lockTime, redeemerPubKeyHash, decodedHashOfSecret); + + Bitcoiny bitcoiny = (Bitcoiny) blockchain.getInstance(); + + return bitcoiny.deriveP2shAddress(redeemScript); + } + + @GET + @Path("/status/{blockchain}/{refundPKH}/{locktime}/{redeemPKH}/{hashOfSecret}") + @Operation( + summary = "Checks HTLC status", + description = "Public key hashes (PKH) and hash of secret should be 20 bytes (base58 encoded). Locktime is seconds since epoch.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = CrossChainBitcoinyHTLCStatus.class)) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) + @SecurityRequirement(name = "apiKey") + public CrossChainBitcoinyHTLCStatus checkHtlcStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("blockchain") String blockchainName, + @PathParam("refundPKH") String refundPKH, + @PathParam("locktime") int lockTime, + @PathParam("redeemPKH") String redeemPKH, + @PathParam("hashOfSecret") String hashOfSecret) { + Security.checkApiCallAllowed(request); + + SupportedBlockchain blockchain = SupportedBlockchain.valueOf(blockchainName); + if (blockchain == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + byte[] refunderPubKeyHash; + byte[] redeemerPubKeyHash; + byte[] decodedHashOfSecret; + + try { + refunderPubKeyHash = Base58.decode(refundPKH); + redeemerPubKeyHash = Base58.decode(redeemPKH); + + if (refunderPubKeyHash.length != 20 || redeemerPubKeyHash.length != 20) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + } catch (IllegalArgumentException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + } + + try { + decodedHashOfSecret = Base58.decode(hashOfSecret); + if (decodedHashOfSecret.length != 20) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } catch (IllegalArgumentException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + byte[] redeemScript = BitcoinyHTLC.buildScript(refunderPubKeyHash, lockTime, redeemerPubKeyHash, decodedHashOfSecret); + + Bitcoiny bitcoiny = (Bitcoiny) blockchain.getInstance(); + + String p2shAddress = bitcoiny.deriveP2shAddress(redeemScript); + + long now = NTP.getTime(); + + try { + int medianBlockTime = bitcoiny.getMedianBlockTime(); + + // Check P2SH is funded + long p2shBalance = bitcoiny.getConfirmedBalance(p2shAddress.toString()); + + CrossChainBitcoinyHTLCStatus htlcStatus = new CrossChainBitcoinyHTLCStatus(); + htlcStatus.bitcoinP2shAddress = p2shAddress; + htlcStatus.bitcoinP2shBalance = BigDecimal.valueOf(p2shBalance, 8); + + List fundingOutputs = bitcoiny.getUnspentOutputs(p2shAddress.toString()); + + if (p2shBalance > 0L && !fundingOutputs.isEmpty()) { + htlcStatus.canRedeem = now >= medianBlockTime * 1000L; + htlcStatus.canRefund = now >= lockTime * 1000L; + } + + if (now >= medianBlockTime * 1000L) { + // See if we can extract secret + htlcStatus.secret = BitcoinyHTLC.findHtlcSecret(bitcoiny, htlcStatus.bitcoinP2shAddress); + } + + return htlcStatus; + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + } + + @POST + @Path("/redeem/{ataddress}") + @Operation( + summary = "Redeems HTLC associated with supplied AT", + description = "To be used by a QORT seller (Bob) who needs to redeem LTC/DOGE/etc proceeds that are stuck in a P2SH.
" + + "This requires Bob's trade bot data to be present in the database for this AT.
" + + "It will fail if the buyer has yet to redeem the QORT held in the AT.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) + @SecurityRequirement(name = "apiKey") + public boolean redeemHtlc(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("ataddress") String atAddress) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + if (crossChainTradeData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Attempt to find secret from the buyer's message to AT + byte[] decodedSecret = acct.findSecretA(repository, crossChainTradeData); + if (decodedSecret == null) { + LOGGER.info(() -> String.format("Unable to find secret-A from redeem message to AT %s", atAddress)); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + TradeBotData tradeBotData = allTradeBotData.stream().filter(tradeBotDataItem -> tradeBotDataItem.getAtAddress().equals(atAddress)).findFirst().orElse(null); + + // Search for the tradePrivateKey in the tradebot data + byte[] decodedPrivateKey = null; + if (tradeBotData != null) + decodedPrivateKey = tradeBotData.getTradePrivateKey(); + + // Search for the foreign blockchain receiving address in the tradebot data + byte[] foreignBlockchainReceivingAccountInfo = null; + if (tradeBotData != null) + // Use receiving address PKH from tradebot data + foreignBlockchainReceivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + + return this.doRedeemHtlc(atAddress, decodedPrivateKey, decodedSecret, foreignBlockchainReceivingAccountInfo); + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST + @Path("/redeemAll") + @Operation( + summary = "Redeems HTLC for all applicable ATs in tradebot data", + description = "To be used by a QORT seller (Bob) who needs to redeem LTC/DOGE/etc proceeds that are stuck in P2SH transactions.
" + + "This requires Bob's trade bot data to be present in the database for any ATs that need redeeming.
" + + "Returns true if at least one trade is redeemed. More detail is available in the log.txt.* file.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) + @SecurityRequirement(name = "apiKey") + public boolean redeemAllHtlc(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + boolean success = false; + + try (final Repository repository = RepositoryManager.getRepository()) { + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + + for (TradeBotData tradeBotData : allTradeBotData) { + String atAddress = tradeBotData.getAtAddress(); + if (atAddress == null) { + LOGGER.info("Missing AT address in tradebot data", atAddress); + continue; + } + + String tradeState = tradeBotData.getState(); + if (tradeState == null) { + LOGGER.info("Missing trade state for AT {}", atAddress); + continue; + } + + if (tradeState.startsWith("ALICE")) { + LOGGER.info("AT {} isn't redeemable because it is a buy order", atAddress); + continue; + } + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) { + LOGGER.info("Couldn't find AT with address {}", atAddress); + continue; + } + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) { + continue; + } + + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + if (crossChainTradeData == null) { + LOGGER.info("Couldn't find crosschain trade data for AT {}", atAddress); + continue; + } + + // Attempt to find secret from the buyer's message to AT + byte[] decodedSecret = acct.findSecretA(repository, crossChainTradeData); + if (decodedSecret == null) { + LOGGER.info("Unable to find secret-A from redeem message to AT {}", atAddress); + continue; + } + + // Search for the tradePrivateKey in the tradebot data + byte[] decodedPrivateKey = tradeBotData.getTradePrivateKey(); + + // Search for the foreign blockchain receiving address PKH in the tradebot data + byte[] foreignBlockchainReceivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + + try { + LOGGER.info("Attempting to redeem P2SH balance associated with AT {}...", atAddress); + boolean redeemed = this.doRedeemHtlc(atAddress, decodedPrivateKey, decodedSecret, foreignBlockchainReceivingAccountInfo); + if (redeemed) { + LOGGER.info("Redeemed P2SH balance associated with AT {}", atAddress); + success = true; + } + else { + LOGGER.info("Couldn't redeem P2SH balance associated with AT {}. Already redeemed?", atAddress); + } + } catch (ApiException e) { + LOGGER.info("Couldn't redeem P2SH balance associated with AT {}. Missing data?", atAddress); + } + } + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + + return success; + } + + private boolean doRedeemHtlc(String atAddress, byte[] decodedTradePrivateKey, byte[] decodedSecret, + byte[] foreignBlockchainReceivingAccountInfo) { + try (final Repository repository = RepositoryManager.getRepository()) { + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + if (crossChainTradeData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Validate trade private key + if (decodedTradePrivateKey == null || decodedTradePrivateKey.length != 32) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Validate secret + if (decodedSecret == null || decodedSecret.length != 32) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Validate receiving address + if (foreignBlockchainReceivingAccountInfo == null || foreignBlockchainReceivingAccountInfo.length != 20) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Make sure the receiving address isn't a QORT address, given that we can share the same field for both QORT and foreign blockchains + if (Crypto.isValidAddress(foreignBlockchainReceivingAccountInfo)) + if (Base58.encode(foreignBlockchainReceivingAccountInfo).startsWith("Q")) + // This is likely a QORT address, not a foreign blockchain + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + + // Use secret-A to redeem P2SH-A + + Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain(); + if (bitcoiny.getClass() == Bitcoin.class) { + LOGGER.info("Redeeming a Bitcoin HTLC is not yet supported"); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + int lockTime = crossChainTradeData.lockTimeA; + byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTime, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); + String p2shAddressA = bitcoiny.deriveP2shAddress(redeemScriptA); + LOGGER.info(String.format("Redeeming P2SH address: %s", p2shAddressA)); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTime, crossChainTradeData.tradeTimeout); + long p2shFee = bitcoiny.getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoiny.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund + return false; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Double-check that we have redeemed P2SH-A... + return false; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // Wait for AT to auto-refund + return false; + + case FUNDED: { + Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey redeemKey = ECKey.fromPrivate(decodedTradePrivateKey); + List fundingOutputs = bitcoiny.getUnspentOutputs(p2shAddressA); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(bitcoiny.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptA, decodedSecret, foreignBlockchainReceivingAccountInfo); + + bitcoiny.broadcastTransaction(p2shRedeemTransaction); + LOGGER.info(String.format("P2SH address %s redeemed!", p2shAddressA)); + return true; + } + } + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, e); + } + + return false; + } + + @POST + @Path("/refund/{ataddress}") + @Operation( + summary = "Refunds HTLC associated with supplied AT", + description = "To be used by a QORT buyer (Alice) who needs to refund their LTC/DOGE/etc that is stuck in a P2SH.
" + + "This requires Alice's trade bot data to be present in the database for this AT.
" + + "It will fail if it's already redeemed by the seller, or if the lockTime (60 minutes) hasn't passed yet.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) + @SecurityRequirement(name = "apiKey") + public boolean refundHtlc(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("ataddress") String atAddress) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + TradeBotData tradeBotData = allTradeBotData.stream().filter(tradeBotDataItem -> tradeBotDataItem.getAtAddress().equals(atAddress)).findFirst().orElse(null); + if (tradeBotData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + if (tradeBotData.getForeignKey() == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Determine foreign blockchain receive address for refund + Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain(); + String receiveAddress = bitcoiny.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + + return this.doRefundHtlc(atAddress, receiveAddress); + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, e); + } + } + + + @POST + @Path("/refundAll") + @Operation( + summary = "Refunds HTLC for all applicable ATs in tradebot data", + description = "To be used by a QORT buyer (Alice) who needs to refund their LTC/DOGE/etc proceeds that are stuck in P2SH transactions.
" + + "This requires Alice's trade bot data to be present in the database for this AT.
" + + "It will fail if it's already redeemed by the seller, or if the lockTime (60 minutes) hasn't passed yet.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) + @SecurityRequirement(name = "apiKey") + public boolean refundAllHtlc(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + boolean success = false; + + try (final Repository repository = RepositoryManager.getRepository()) { + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + + for (TradeBotData tradeBotData : allTradeBotData) { + String atAddress = tradeBotData.getAtAddress(); + if (atAddress == null) { + LOGGER.info("Missing AT address in tradebot data", atAddress); + continue; + } + + String tradeState = tradeBotData.getState(); + if (tradeState == null) { + LOGGER.info("Missing trade state for AT {}", atAddress); + continue; + } + + if (tradeState.startsWith("BOB")) { + LOGGER.info("AT {} isn't refundable because it is a sell order", atAddress); + continue; + } + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) { + LOGGER.info("Couldn't find AT with address {}", atAddress); + continue; + } + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) { + continue; + } + + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + if (crossChainTradeData == null) { + LOGGER.info("Couldn't find crosschain trade data for AT {}", atAddress); + continue; + } + + if (tradeBotData.getForeignKey() == null) { + LOGGER.info("Couldn't find foreign key for AT {}", atAddress); + continue; + } + + try { + // Determine foreign blockchain receive address for refund + Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain(); + String receivingAddress = bitcoiny.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + + LOGGER.info("Attempting to refund P2SH balance associated with AT {}...", atAddress); + boolean refunded = this.doRefundHtlc(atAddress, receivingAddress); + if (refunded) { + LOGGER.info("Refunded P2SH balance associated with AT {}", atAddress); + success = true; + } + else { + LOGGER.info("Couldn't refund P2SH balance associated with AT {}. Already redeemed?", atAddress); + } + } catch (ApiException | ForeignBlockchainException e) { + LOGGER.info("Couldn't refund P2SH balance associated with AT {}. Missing data?", atAddress); + } + } + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + + return success; + } + + + private boolean doRefundHtlc(String atAddress, String receiveAddress) { + try (final Repository repository = RepositoryManager.getRepository()) { + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + if (crossChainTradeData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // If the AT is "finished" then it will have a zero balance + // In these cases we should avoid HTLC refunds if tbe QORT haven't been returned to the seller + if (atData.getIsFinished() && crossChainTradeData.mode != AcctMode.REFUNDED && crossChainTradeData.mode != AcctMode.CANCELLED) { + LOGGER.info(String.format("Skipping AT %s because the QORT has already been redemed", atAddress)); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + TradeBotData tradeBotData = allTradeBotData.stream().filter(tradeBotDataItem -> tradeBotDataItem.getAtAddress().equals(atAddress)).findFirst().orElse(null); + if (tradeBotData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain(); + if (bitcoiny.getClass() == Bitcoin.class) { + LOGGER.info("Refunding a Bitcoin HTLC is not yet supported"); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + int lockTime = tradeBotData.getLockTimeA(); + + // We can't refund P2SH-A until lockTime-A has passed + if (NTP.getTime() <= lockTime * 1000L) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_TOO_SOON); + + // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) + int medianBlockTime = bitcoiny.getMedianBlockTime(); + if (medianBlockTime <= lockTime) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_TOO_SOON); + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTime, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = bitcoiny.deriveP2shAddress(redeemScriptA); + LOGGER.info(String.format("Refunding P2SH address: %s", p2shAddressA)); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTime, crossChainTradeData.tradeTimeout); + long p2shFee = bitcoiny.getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoiny.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-A to be funded... + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_TOO_SOON); + + case REDEEM_IN_PROGRESS: + case REDEEMED: + case REFUND_IN_PROGRESS: + case REFUNDED: + // Too late! + return false; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = bitcoiny.getUnspentOutputs(p2shAddressA); + + // Validate the destination foreign blockchain address + Address receiving = Address.fromString(bitcoiny.getNetworkParameters(), receiveAddress); + if (receiving.getOutputScriptType() != Script.ScriptType.P2PKH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoiny.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptA, lockTime, receiving.getHash()); + + bitcoiny.broadcastTransaction(p2shRefundTransaction); + return true; + } + } + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, e); + } + + return false; + } + + private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { + return (lockTimeA - tradeTimeout * 60) * 1000L; + } + +} diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java new file mode 100644 index 000000000..7b6bc962a --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java @@ -0,0 +1,148 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.CrossChainSecretRequest; +import org.qortal.crosschain.AcctMode; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.Transformer; +import org.qortal.transform.transaction.MessageTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.HeaderParam; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import java.util.Arrays; +import java.util.Random; + +@Path("/crosschain/LitecoinACCTv1") +@Tag(name = "Cross-Chain (LitecoinACCTv1)") +public class CrossChainLitecoinACCTv1Resource { + + @Context + HttpServletRequest request; + + @POST + @Path("/redeemmessage") + @Operation( + summary = "Signs and broadcasts a 'redeem' MESSAGE transaction that sends secrets to AT, releasing funds to partner", + description = "Specify address of cross-chain AT that needs to be messaged, Alice's trade private key, the 32-byte secret,
" + + "and an address for receiving QORT from AT. All of these can be found in Alice's trade bot data.
" + + "AT needs to be in 'trade' mode. Messages sent to an AT in any other mode will be ignored, but still cost fees to send!
" + + "You need to use the private key that the AT considers the trade 'partner' otherwise the MESSAGE transaction will be invalid.", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = CrossChainSecretRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public boolean buildRedeemMessage(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainSecretRequest secretRequest) { + Security.checkApiCallAllowed(request); + + byte[] partnerPrivateKey = secretRequest.partnerPrivateKey; + + if (partnerPrivateKey == null || partnerPrivateKey.length != Transformer.PRIVATE_KEY_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + if (secretRequest.atAddress == null || !Crypto.isValidAtAddress(secretRequest.atAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (secretRequest.secret == null || secretRequest.secret.length != LitecoinACCTv1.SECRET_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (secretRequest.receivingAddress == null || !Crypto.isValidAddress(secretRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = fetchAtDataWithChecking(repository, secretRequest.atAddress); + CrossChainTradeData crossChainTradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + + if (crossChainTradeData.mode != AcctMode.TRADING) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + byte[] partnerPublicKey = new PrivateKeyAccount(null, partnerPrivateKey).getPublicKey(); + String partnerAddress = Crypto.toAddress(partnerPublicKey); + + // MESSAGE must come from address that AT considers trade partner + if (!crossChainTradeData.qortalPartnerAddress.equals(partnerAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + // Good to make MESSAGE + + byte[] messageData = LitecoinACCTv1.buildRedeemMessage(secretRequest.secret, secretRequest.receivingAddress); + + PrivateKeyAccount sender = new PrivateKeyAccount(repository, partnerPrivateKey); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, secretRequest.atAddress, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID); + + return true; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + // Must be correct AT - check functionality using code hash + if (!Arrays.equals(atData.getCodeHash(), LitecoinACCTv1.CODE_BYTES_HASH)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // No point sending message to AT that's finished + if (atData.getIsFinished()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + return atData; + } + +} diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java new file mode 100644 index 000000000..627c00c7f --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java @@ -0,0 +1,177 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; + +import java.util.List; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.HeaderParam; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + +import org.bitcoinj.core.Transaction; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.crosschain.LitecoinSendRequest; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.SimpleTransaction; + +@Path("/crosschain/ltc") +@Tag(name = "Cross-Chain (Litecoin)") +public class CrossChainLitecoinResource { + + @Context + HttpServletRequest request; + + @POST + @Path("/walletbalance") + @Operation( + summary = "Returns LTC balance for hierarchical, deterministic BIP32 wallet", + description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + description = "BIP32 'm' private/public key in base58", + example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "balance (satoshis)")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String getLitecoinWalletBalance(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) { + Security.checkApiCallAllowed(request); + + Litecoin litecoin = Litecoin.getInstance(); + + if (!litecoin.isValidDeterministicKey(key58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + try { + Long balance = litecoin.getWalletBalanceFromTransactions(key58); + if (balance == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + + return balance.toString(); + + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + } + + @POST + @Path("/wallettransactions") + @Operation( + summary = "Returns transactions for hierarchical, deterministic BIP32 wallet", + description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + description = "BIP32 'm' private/public key in base58", + example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(array = @ArraySchema( schema = @Schema( implementation = SimpleTransaction.class ) ) ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + @SecurityRequirement(name = "apiKey") + public List getLitecoinWalletTransactions(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) { + Security.checkApiCallAllowed(request); + + Litecoin litecoin = Litecoin.getInstance(); + + if (!litecoin.isValidDeterministicKey(key58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + try { + return litecoin.getWalletTransactions(key58); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + } + + @POST + @Path("/send") + @Operation( + summary = "Sends LTC from hierarchical, deterministic BIP32 wallet to specific address", + description = "Currently only supports 'legacy' P2PKH Litecoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = LitecoinSendRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "transaction hash")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String sendBitcoin(@HeaderParam(Security.API_KEY_HEADER) String apiKey, LitecoinSendRequest litecoinSendRequest) { + Security.checkApiCallAllowed(request); + + if (litecoinSendRequest.litecoinAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + if (litecoinSendRequest.feePerByte != null && litecoinSendRequest.feePerByte <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + Litecoin litecoin = Litecoin.getInstance(); + + if (!litecoin.isValidAddress(litecoinSendRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (!litecoin.isValidDeterministicKey(litecoinSendRequest.xprv58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + Transaction spendTransaction = litecoin.buildSpend(litecoinSendRequest.xprv58, + litecoinSendRequest.receivingAddress, + litecoinSendRequest.litecoinAmount, + litecoinSendRequest.feePerByte); + + if (spendTransaction == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE); + + try { + litecoin.broadcastTransaction(spendTransaction); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + + return spendTransaction.getTxId().toString(); + } + +} diff --git a/src/main/java/org/qortal/api/resource/CrossChainResource.java b/src/main/java/org/qortal/api/resource/CrossChainResource.java index f60deb233..47eee3010 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainResource.java @@ -1,5 +1,6 @@ package org.qortal.api.resource; +import com.google.common.primitives.Longs; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.ArraySchema; @@ -7,69 +8,47 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import java.util.*; +import java.util.function.Supplier; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.QueryParam; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.ECKey; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.core.TransactionOutput; -import org.qortal.account.PublicKeyAccount; import org.qortal.api.ApiError; import org.qortal.api.ApiErrors; -import org.qortal.api.ApiException; import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; import org.qortal.api.model.CrossChainCancelRequest; -import org.qortal.api.model.CrossChainSecretRequest; -import org.qortal.api.model.CrossChainTradeRequest; -import org.qortal.api.model.CrossChainBitcoinP2SHStatus; -import org.qortal.api.model.CrossChainBitcoinRedeemRequest; -import org.qortal.api.model.CrossChainBitcoinRefundRequest; -import org.qortal.api.model.CrossChainBitcoinTemplateRequest; -import org.qortal.api.model.CrossChainBuildRequest; -import org.qortal.asset.Asset; -import org.qortal.crosschain.BTC; -import org.qortal.crosschain.BTCACCT; +import org.qortal.api.model.CrossChainTradeSummary; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.AcctMode; import org.qortal.crypto.Crypto; import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.crosschain.CrossChainTradeData.Mode; import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; import org.qortal.data.transaction.MessageTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.group.Group; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; -import org.qortal.transaction.DeployAtTransaction; import org.qortal.transaction.MessageTransaction; -import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.ValidationResult; import org.qortal.transform.TransformationException; import org.qortal.transform.Transformer; -import org.qortal.transform.transaction.DeployAtTransactionTransformer; import org.qortal.transform.transaction.MessageTransactionTransformer; +import org.qortal.utils.Amounts; import org.qortal.utils.Base58; +import org.qortal.utils.ByteArray; import org.qortal.utils.NTP; -import com.google.common.primitives.Bytes; - @Path("/crosschain") @Tag(name = "Cross-Chain") public class CrossChainResource { @@ -83,7 +62,6 @@ public class CrossChainResource { summary = "Find cross-chain trade offers", responses = { @ApiResponse( - description = "automated transactions", content = @Content( array = @ArraySchema( schema = @Schema( @@ -96,6 +74,11 @@ public class CrossChainResource { ) @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) public List getTradeOffers( + @Parameter( + description = "Limit to specific blockchain", + example = "LITECOIN", + schema = @Schema(implementation = SupportedBlockchain.class) + ) @QueryParam("foreignBlockchain") SupportedBlockchain foreignBlockchain, @Parameter( ref = "limit") @QueryParam("limit") Integer limit, @Parameter( ref = "offset" ) @QueryParam("offset") Integer offset, @Parameter( ref = "reverse" ) @QueryParam("reverse") Boolean reverse) { @@ -103,633 +86,356 @@ public List getTradeOffers( if (limit != null && limit > 100) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - byte[] codeHash = BTCACCT.CODE_BYTES_HASH; - boolean isExecutable = true; - - try (final Repository repository = RepositoryManager.getRepository()) { - List atsData = repository.getATRepository().getATsByFunctionality(codeHash, isExecutable, limit, offset, reverse); - - List crossChainTradesData = new ArrayList<>(); - for (ATData atData : atsData) { - CrossChainTradeData crossChainTradeData = BTCACCT.populateTradeData(repository, atData); - crossChainTradesData.add(crossChainTradeData); - } - - return crossChainTradesData; - } catch (ApiException e) { - throw e; - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @POST - @Path("/build") - @Operation( - summary = "Build cross-chain trading AT", - description = "Returns raw, unsigned DEPLOY_AT transaction", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = CrossChainBuildRequest.class - ) - ) - ), - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) - ) - } - ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_DATA, ApiError.INVALID_REFERENCE, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE}) - public String buildTrade(CrossChainBuildRequest tradeRequest) { - byte[] creatorPublicKey = tradeRequest.creatorPublicKey; - - if (creatorPublicKey == null || creatorPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - if (tradeRequest.secretHash == null || tradeRequest.secretHash.length != BTC.HASH160_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - if (tradeRequest.tradeTimeout == null) - tradeRequest.tradeTimeout = 7 * 24 * 60; // 7 days - else - if (tradeRequest.tradeTimeout < 10 || tradeRequest.tradeTimeout > 50000) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - if (tradeRequest.initialQortAmount < 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - if (tradeRequest.finalQortAmount <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - if (tradeRequest.fundingQortAmount <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - // funding amount must exceed initial + final - if (tradeRequest.fundingQortAmount <= tradeRequest.initialQortAmount + tradeRequest.finalQortAmount) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - if (tradeRequest.bitcoinAmount <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + final boolean isExecutable = true; + List crossChainTrades = new ArrayList<>(); try (final Repository repository = RepositoryManager.getRepository()) { - PublicKeyAccount creatorAccount = new PublicKeyAccount(repository, creatorPublicKey); + Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(foreignBlockchain); - byte[] creationBytes = BTCACCT.buildQortalAT(creatorAccount.getAddress(), tradeRequest.secretHash, tradeRequest.tradeTimeout, tradeRequest.initialQortAmount, tradeRequest.finalQortAmount, tradeRequest.bitcoinAmount); + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); - long txTimestamp = NTP.getTime(); - byte[] lastReference = creatorAccount.getLastReference(); - if (lastReference == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_REFERENCE); + List atsData = repository.getATRepository().getATsByFunctionality(codeHash, isExecutable, limit, offset, reverse); - long fee = 0; - String name = "QORT-BTC cross-chain trade"; - String description = "Qortal-Bitcoin cross-chain trade"; - String atType = "ACCT"; - String tags = "QORT-BTC ACCT"; - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, creatorAccount.getPublicKey(), fee, null); - TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, tradeRequest.fundingQortAmount, Asset.QORT); - - Transaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + for (ATData atData : atsData) { + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + if (crossChainTradeData.mode == AcctMode.OFFERING) { + crossChainTrades.add(crossChainTradeData); + } + } + } - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); + // Sort the trades by timestamp + if (reverse != null && reverse) { + crossChainTrades.sort((a, b) -> Longs.compare(b.creationTimestamp, a.creationTimestamp)); + } + else { + crossChainTrades.sort((a, b) -> Longs.compare(a.creationTimestamp, b.creationTimestamp)); + } - ValidationResult result = deployAtTransaction.isValidUnconfirmed(); - if (result != ValidationResult.OK) - throw TransactionsResource.createTransactionInvalidException(request, result); + if (limit != null && limit > 0) { + // Make sure to not return more than the limit + int upperLimit = Math.min(limit, crossChainTrades.size()); + crossChainTrades = crossChainTrades.subList(0, upperLimit); + } - byte[] bytes = DeployAtTransactionTransformer.toBytes(deployAtTransactionData); - return Base58.encode(bytes); - } catch (TransformationException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + return crossChainTrades; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } } - @POST - @Path("/tradeoffer/recipient") + @GET + @Path("/trade/{ataddress}") @Operation( - summary = "Builds raw, unsigned MESSAGE transaction that sends cross-chain trade recipient address, triggering 'trade' mode", - description = "Specify address of cross-chain AT that needs to be messaged, and address of Qortal recipient.
" - + "AT needs to be in 'offer' mode. Messages sent to an AT in 'trade' mode will be ignored, but still cost fees to send!
" - + "You need to sign output with same account as the AT creator otherwise the MESSAGE transaction will be invalid.", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = CrossChainTradeRequest.class - ) - ) - ), + summary = "Show detailed trade info", responses = { @ApiResponse( content = @Content( schema = @Schema( - type = "string" + implementation = CrossChainTradeData.class ) ) ) } ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) - public String sendTradeRecipient(CrossChainTradeRequest tradeRequest) { - byte[] creatorPublicKey = tradeRequest.creatorPublicKey; - - if (creatorPublicKey == null || creatorPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - if (tradeRequest.atAddress == null || !Crypto.isValidAtAddress(tradeRequest.atAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - if (tradeRequest.recipient == null || !Crypto.isValidAddress(tradeRequest.recipient)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - + @ApiErrors({ApiError.ADDRESS_UNKNOWN, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public CrossChainTradeData getTrade(@PathParam("ataddress") String atAddress) { try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = fetchAtDataWithChecking(repository, creatorPublicKey, tradeRequest.atAddress); - CrossChainTradeData crossChainTradeData = BTCACCT.populateTradeData(repository, atData); + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - if (crossChainTradeData.mode == CrossChainTradeData.Mode.TRADE) + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - // Good to make MESSAGE - - byte[] recipientAddressBytes = Bytes.ensureCapacity(Base58.decode(tradeRequest.recipient), 32, 0); - byte[] messageTransactionBytes = buildAtMessage(repository, creatorPublicKey, tradeRequest.atAddress, recipientAddressBytes); - - return Base58.encode(messageTransactionBytes); + return acct.populateTradeData(repository, atData); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } } - @POST - @Path("/tradeoffer/secret") + @GET + @Path("/trades") @Operation( - summary = "Builds raw, unsigned MESSAGE transaction that sends secret to AT, releasing funds to recipient", - description = "Specify address of cross-chain AT that needs to be messaged, and 32-byte secret.
" - + "AT needs to be in 'trade' mode. Messages sent to an AT in 'trade' mode will be ignored, but still cost fees to send!
" - + "You need to sign output with account the AT considers the 'recipient' otherwise the MESSAGE transaction will be invalid.", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = CrossChainSecretRequest.class - ) - ) - ), + summary = "Find completed cross-chain trades", + description = "Returns summary info about successfully completed cross-chain trades", responses = { @ApiResponse( content = @Content( - schema = @Schema( - type = "string" + array = @ArraySchema( + schema = @Schema( + implementation = CrossChainTradeSummary.class + ) ) ) ) } ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) - public String sendSecret(CrossChainSecretRequest secretRequest) { - byte[] recipientPublicKey = secretRequest.recipientPublicKey; - - if (recipientPublicKey == null || recipientPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public List getCompletedTrades( + @Parameter( + description = "Limit to specific blockchain", + example = "LITECOIN", + schema = @Schema(implementation = SupportedBlockchain.class) + ) @QueryParam("foreignBlockchain") SupportedBlockchain foreignBlockchain, + @Parameter( + description = "Only return trades that completed on/after this timestamp (milliseconds since epoch)", + example = "1597310000000" + ) @QueryParam("minimumTimestamp") Long minimumTimestamp, + @Parameter( ref = "limit") @QueryParam("limit") Integer limit, + @Parameter( ref = "offset" ) @QueryParam("offset") Integer offset, + @Parameter( ref = "reverse" ) @QueryParam("reverse") Boolean reverse) { + // Impose a limit on 'limit' + if (limit != null && limit > 100) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - if (secretRequest.atAddress == null || !Crypto.isValidAtAddress(secretRequest.atAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + // minimumTimestamp (if given) needs to be positive + if (minimumTimestamp != null && minimumTimestamp <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - if (secretRequest.secret == null || secretRequest.secret.length != BTCACCT.SECRET_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + final Boolean isFinished = Boolean.TRUE; try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = fetchAtDataWithChecking(repository, null, secretRequest.atAddress); // null to skip creator check - CrossChainTradeData crossChainTradeData = BTCACCT.populateTradeData(repository, atData); + Integer minimumFinalHeight = null; + + if (minimumTimestamp != null) { + minimumFinalHeight = repository.getBlockRepository().getHeightFromTimestamp(minimumTimestamp); + // If not found in the block repository it will return either 0 or 1 + if (minimumFinalHeight == 0 || minimumFinalHeight == 1) { + // Try the archive + minimumFinalHeight = repository.getBlockArchiveRepository().getHeightFromTimestamp(minimumTimestamp); + } + + if (minimumFinalHeight == 0) + // We don't have any blocks since minimumTimestamp, let alone trades, so nothing to return + return Collections.emptyList(); + + // height returned from repository is for block BEFORE timestamp + // but we want trades AFTER timestamp so bump height accordingly + minimumFinalHeight++; + } - if (crossChainTradeData.mode == CrossChainTradeData.Mode.OFFER) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + List crossChainTrades = new ArrayList<>(); - PublicKeyAccount recipientAccount = new PublicKeyAccount(repository, recipientPublicKey); - String recipientAddress = recipientAccount.getAddress(); + Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(foreignBlockchain); - // MESSAGE must come from address that AT considers trade partner / 'recipient' - if (!crossChainTradeData.qortalRecipient.equals(recipientAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); - // Good to make MESSAGE + List atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, + isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumFinalHeight, + limit, offset, reverse); - byte[] messageTransactionBytes = buildAtMessage(repository, recipientPublicKey, secretRequest.atAddress, secretRequest.secret); + for (ATStateData atState : atStates) { + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState); - return Base58.encode(messageTransactionBytes); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } + // We also need block timestamp for use as trade timestamp + long timestamp = repository.getBlockRepository().getTimestampFromHeight(atState.getHeight()); + if (timestamp == 0) { + // Try the archive + timestamp = repository.getBlockArchiveRepository().getTimestampFromHeight(atState.getHeight()); + } - @DELETE - @Path("/tradeoffer") - @Operation( - summary = "Builds raw, unsigned MESSAGE transaction that cancels cross-chain trade offer", - description = "Specify address of cross-chain AT that needs to be cancelled.
" - + "AT needs to be in 'offer' mode. Messages sent to an AT in 'trade' mode will be ignored, but still cost fees to send!
" - + "You need to sign output with same account as the AT creator otherwise the MESSAGE transaction will be invalid.", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = CrossChainCancelRequest.class - ) - ) - ), - responses = { - @ApiResponse( - content = @Content( - schema = @Schema( - type = "string" - ) - ) - ) - } - ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) - public String cancelTradeOffer(CrossChainCancelRequest cancelRequest) { - byte[] creatorPublicKey = cancelRequest.creatorPublicKey; - - if (creatorPublicKey == null || creatorPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - if (cancelRequest.atAddress == null || !Crypto.isValidAtAddress(cancelRequest.atAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = fetchAtDataWithChecking(repository, creatorPublicKey, cancelRequest.atAddress); - CrossChainTradeData crossChainTradeData = BTCACCT.populateTradeData(repository, atData); - - if (crossChainTradeData.mode == CrossChainTradeData.Mode.TRADE) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // Good to make MESSAGE + CrossChainTradeSummary crossChainTradeSummary = new CrossChainTradeSummary(crossChainTradeData, timestamp); + crossChainTrades.add(crossChainTradeSummary); + } + } - PublicKeyAccount creatorAccount = new PublicKeyAccount(repository, creatorPublicKey); - String creatorAddress = creatorAccount.getAddress(); - byte[] recipientAddressBytes = Bytes.ensureCapacity(Base58.decode(creatorAddress), 32, 0); + // Sort the trades by timestamp + if (reverse != null && reverse) { + crossChainTrades.sort((a, b) -> Longs.compare(b.getTradeTimestamp(), a.getTradeTimestamp())); + } + else { + crossChainTrades.sort((a, b) -> Longs.compare(a.getTradeTimestamp(), b.getTradeTimestamp())); + } - byte[] messageTransactionBytes = buildAtMessage(repository, creatorPublicKey, cancelRequest.atAddress, recipientAddressBytes); + if (limit != null && limit > 0) { + // Make sure to not return more than the limit + int upperLimit = Math.min(limit, crossChainTrades.size()); + crossChainTrades = crossChainTrades.subList(0, upperLimit); + } - return Base58.encode(messageTransactionBytes); + return crossChainTrades; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } } - @POST - @Path("/p2sh") + @GET + @Path("/price/{blockchain}") @Operation( - summary = "Returns Bitcoin P2SH address based on trade info", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = CrossChainBitcoinTemplateRequest.class - ) - ) - ), + summary = "Request current estimated trading price", + description = "Returns price based on most recent completed trades. Price is expressed in terms of QORT per unit foreign currency.", responses = { @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) - ) - } - ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE}) - public String deriveP2sh(CrossChainBitcoinTemplateRequest templateRequest) { - BTC btc = BTC.getInstance(); - NetworkParameters params = btc.getNetworkParameters(); - - if (templateRequest.refundPublicKeyHash == null || templateRequest.refundPublicKeyHash.length != 20) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - if (templateRequest.redeemPublicKeyHash == null || templateRequest.redeemPublicKeyHash.length != 20) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - if (templateRequest.atAddress == null || !Crypto.isValidAtAddress(templateRequest.atAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - // Extract data from cross-chain trading AT - try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = fetchAtDataWithChecking(repository, null, templateRequest.atAddress); // null to skip creator check - CrossChainTradeData crossChainTradeData = BTCACCT.populateTradeData(repository, atData); - - if (crossChainTradeData.mode == Mode.OFFER) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - byte[] redeemScriptBytes = BTCACCT.buildScript(templateRequest.refundPublicKeyHash, crossChainTradeData.lockTime, templateRequest.redeemPublicKeyHash, crossChainTradeData.secretHash); - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - - Address p2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - return p2shAddress.toString(); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @POST - @Path("/p2sh/check") - @Operation( - summary = "Checks Bitcoin P2SH address based on trade info", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = CrossChainBitcoinTemplateRequest.class + content = @Content( + schema = @Schema( + type = "number" + ) ) ) - ), - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = CrossChainBitcoinP2SHStatus.class)) - ) } ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN, ApiError.REPOSITORY_ISSUE}) - public CrossChainBitcoinP2SHStatus checkP2sh(CrossChainBitcoinTemplateRequest templateRequest) { - BTC btc = BTC.getInstance(); - NetworkParameters params = btc.getNetworkParameters(); - - if (templateRequest.refundPublicKeyHash == null || templateRequest.refundPublicKeyHash.length != 20) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - if (templateRequest.redeemPublicKeyHash == null || templateRequest.redeemPublicKeyHash.length != 20) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public long getTradePriceEstimate( + @Parameter( + description = "foreign blockchain", + example = "LITECOIN", + schema = @Schema(implementation = SupportedBlockchain.class) + ) @PathParam("blockchain") SupportedBlockchain foreignBlockchain, + @Parameter( + description = "Maximum number of trades to include in price calculation", + example = "10", + schema = @Schema(type = "integer", defaultValue = "10") + ) @QueryParam("maxtrades") Integer maxtrades, + @Parameter( + description = "Display price in terms of foreign currency per unit QORT", + example = "false", + schema = @Schema(type = "boolean", defaultValue = "false") + ) @QueryParam("inverse") Boolean inverse) { + // foreignBlockchain is required + if (foreignBlockchain == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - if (templateRequest.atAddress == null || !Crypto.isValidAtAddress(templateRequest.atAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + // We want both a minimum of 5 trades and enough trades to span at least 4 hours + int minimumCount = 5; + int maximumCount = maxtrades != null ? maxtrades : 10; + long minimumPeriod = 4 * 60 * 60 * 1000L; // ms + Boolean isFinished = Boolean.TRUE; + boolean useInversePrice = (inverse != null && inverse == true); - // Extract data from cross-chain trading AT try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = fetchAtDataWithChecking(repository, null, templateRequest.atAddress); // null to skip creator check - CrossChainTradeData crossChainTradeData = BTCACCT.populateTradeData(repository, atData); + Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(foreignBlockchain); - if (crossChainTradeData.mode == Mode.OFFER) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - byte[] redeemScriptBytes = BTCACCT.buildScript(templateRequest.refundPublicKeyHash, crossChainTradeData.lockTime, templateRequest.redeemPublicKeyHash, crossChainTradeData.secretHash); - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - - Address p2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); + long totalForeign = 0; + long totalQort = 0; - Integer medianBlockTime = BTC.getInstance().getMedianBlockTime(); - if (medianBlockTime == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BTC_NETWORK_ISSUE); + Map reverseSortedTradeData = new TreeMap<>(Collections.reverseOrder()); - long now = NTP.getTime(); + // Collect recent AT states for each ACCT version + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); - // Check P2SH is funded - - Coin p2shBalance = BTC.getInstance().getBalance(p2shAddress.toString()); - if (p2shBalance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + List atStates = repository.getATRepository().getMatchingFinalATStatesQuorum(codeHash, + isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumCount, maximumCount, minimumPeriod); - CrossChainBitcoinP2SHStatus p2shStatus = new CrossChainBitcoinP2SHStatus(); - p2shStatus.bitcoinP2shAddress = p2shAddress.toString(); - p2shStatus.bitcoinP2shBalance = BigDecimal.valueOf(p2shBalance.value, 8); + for (ATStateData atState : atStates) { + // We also need block timestamp for use as trade timestamp + long timestamp = repository.getBlockRepository().getTimestampFromHeight(atState.getHeight()); + if (timestamp == 0) { + // Try the archive + timestamp = repository.getBlockArchiveRepository().getTimestampFromHeight(atState.getHeight()); + } - List fundingOutputs = BTC.getInstance().getUnspentOutputs(p2shAddress.toString()); - - if (p2shBalance.value >= crossChainTradeData.expectedBitcoin && !fundingOutputs.isEmpty()) { - p2shStatus.canRedeem = now >= medianBlockTime * 1000L; - p2shStatus.canRefund = now >= crossChainTradeData.lockTime * 1000L; + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState); + reverseSortedTradeData.put(timestamp, crossChainTradeData); + } } - if (now >= medianBlockTime * 1000L) { - // See if we can extract secret - List rawTransactions = BTC.getInstance().getAddressTransactions(p2shStatus.bitcoinP2shAddress); - p2shStatus.secret = BTCACCT.findP2shSecret(p2shStatus.bitcoinP2shAddress, rawTransactions); + // Loop through the sorted map and calculate the average price + // Also remove elements beyond the maxtrades limit + Set set = reverseSortedTradeData.entrySet(); + Iterator i = set.iterator(); + int index = 0; + while (i.hasNext()) { + Map.Entry tradeDataMap = (Map.Entry)i.next(); + CrossChainTradeData crossChainTradeData = (CrossChainTradeData) tradeDataMap.getValue(); + + if (maxtrades != null && index >= maxtrades) { + // We've reached the limit + break; + } + + totalForeign += crossChainTradeData.expectedForeignAmount; + totalQort += crossChainTradeData.qortAmount; + index++; } - return p2shStatus; + return useInversePrice ? Amounts.scaledDivide(totalForeign, totalQort) : Amounts.scaledDivide(totalQort, totalForeign); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } } - @POST - @Path("/p2sh/refund") + @DELETE + @Path("/tradeoffer") @Operation( - summary = "Returns serialized Bitcoin transaction attempting refund from P2SH address", + summary = "Builds raw, unsigned 'cancel' MESSAGE transaction that cancels cross-chain trade offer", + description = "Specify address of cross-chain AT that needs to be cancelled.
" + + "AT needs to be in 'offer' mode. Messages sent to an AT in 'trade' mode will be ignored.
" + + "Performs MESSAGE proof-of-work.
" + + "You need to sign output with AT creator's private key otherwise the MESSAGE transaction will be invalid.", requestBody = @RequestBody( required = true, content = @Content( mediaType = MediaType.APPLICATION_JSON, schema = @Schema( - implementation = CrossChainBitcoinRefundRequest.class + implementation = CrossChainCancelRequest.class ) ) ), responses = { @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + content = @Content( + schema = @Schema( + type = "string" + ) + ) ) } ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN, - ApiError.BTC_TOO_SOON, ApiError.BTC_BALANCE_ISSUE, ApiError.BTC_NETWORK_ISSUE, ApiError.REPOSITORY_ISSUE}) - public String refundP2sh(CrossChainBitcoinRefundRequest refundRequest) { - BTC btc = BTC.getInstance(); - NetworkParameters params = btc.getNetworkParameters(); - - byte[] refundPrivateKey = refundRequest.refundPrivateKey; - if (refundPrivateKey == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - ECKey refundKey = null; + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String cancelTrade(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainCancelRequest cancelRequest) { + Security.checkApiCallAllowed(request); - try { - // Auto-trim - if (refundPrivateKey.length >= 37 && refundPrivateKey.length <= 38) - refundPrivateKey = Arrays.copyOfRange(refundPrivateKey, 1, 33); - if (refundPrivateKey.length != 32) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - refundKey = ECKey.fromPrivate(refundPrivateKey); - } catch (IllegalArgumentException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - } + byte[] creatorPublicKey = cancelRequest.creatorPublicKey; - if (refundRequest.redeemPublicKeyHash == null || refundRequest.redeemPublicKeyHash.length != 20) + if (creatorPublicKey == null || creatorPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - if (refundRequest.atAddress == null || !Crypto.isValidAtAddress(refundRequest.atAddress)) + if (cancelRequest.atAddress == null || !Crypto.isValidAtAddress(cancelRequest.atAddress)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - // Extract data from cross-chain trading AT try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = fetchAtDataWithChecking(repository, null, refundRequest.atAddress); // null to skip creator check - CrossChainTradeData crossChainTradeData = BTCACCT.populateTradeData(repository, atData); - - if (crossChainTradeData.mode == Mode.OFFER) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - byte[] redeemScriptBytes = BTCACCT.buildScript(refundKey.getPubKeyHash(), crossChainTradeData.lockTime, refundRequest.redeemPublicKeyHash, crossChainTradeData.secretHash); - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + ATData atData = fetchAtDataWithChecking(repository, cancelRequest.atAddress); - Address p2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - - long now = NTP.getTime(); - - // Check P2SH is funded - - Coin p2shBalance = BTC.getInstance().getBalance(p2shAddress.toString()); - if (p2shBalance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - - List fundingOutputs = BTC.getInstance().getUnspentOutputs(p2shAddress.toString()); - if (fundingOutputs.isEmpty()) + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - boolean canRefund = now >= crossChainTradeData.lockTime * 1000L; - if (!canRefund) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BTC_TOO_SOON); - - if (p2shBalance.value < crossChainTradeData.expectedBitcoin) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BTC_BALANCE_ISSUE); - - Coin refundAmount = p2shBalance.subtract(Coin.valueOf(refundRequest.bitcoinMinerFee.unscaledValue().longValue())); - - org.bitcoinj.core.Transaction refundTransaction = BTCACCT.buildRefundTransaction(refundAmount, refundKey, fundingOutputs, redeemScriptBytes, crossChainTradeData.lockTime); - boolean wasBroadcast = BTC.getInstance().broadcastTransaction(refundTransaction); - - if (!wasBroadcast) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BTC_NETWORK_ISSUE); + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); - return refundTransaction.getTxId().toString(); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @POST - @Path("/p2sh/redeem") - @Operation( - summary = "Returns serialized Bitcoin transaction attempting redeem from P2SH address", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = CrossChainBitcoinRedeemRequest.class - ) - ) - ), - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) - ) - } - ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN, - ApiError.BTC_TOO_SOON, ApiError.BTC_BALANCE_ISSUE, ApiError.BTC_NETWORK_ISSUE, ApiError.REPOSITORY_ISSUE}) - public String redeemP2sh(CrossChainBitcoinRedeemRequest redeemRequest) { - BTC btc = BTC.getInstance(); - NetworkParameters params = btc.getNetworkParameters(); - - byte[] redeemPrivateKey = redeemRequest.redeemPrivateKey; - if (redeemPrivateKey == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - ECKey redeemKey = null; - - try { - // Auto-trim - if (redeemPrivateKey.length >= 37 && redeemPrivateKey.length <= 38) - redeemPrivateKey = Arrays.copyOfRange(redeemPrivateKey, 1, 33); - if (redeemPrivateKey.length != 32) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - redeemKey = ECKey.fromPrivate(redeemPrivateKey); - } catch (IllegalArgumentException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - } - - if (redeemRequest.refundPublicKeyHash == null || redeemRequest.refundPublicKeyHash.length != 20) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - if (redeemRequest.atAddress == null || !Crypto.isValidAtAddress(redeemRequest.atAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - if (redeemRequest.secret == null || redeemRequest.secret.length != BTCACCT.SECRET_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - // Extract data from cross-chain trading AT - try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = fetchAtDataWithChecking(repository, null, redeemRequest.atAddress); // null to skip creator check - CrossChainTradeData crossChainTradeData = BTCACCT.populateTradeData(repository, atData); - - if (crossChainTradeData.mode == Mode.OFFER) + if (crossChainTradeData.mode != AcctMode.OFFERING) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - byte[] redeemScriptBytes = BTCACCT.buildScript(redeemRequest.refundPublicKeyHash, crossChainTradeData.lockTime, redeemKey.getPubKeyHash(), crossChainTradeData.secretHash); - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - - Address p2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - - Integer medianBlockTime = BTC.getInstance().getMedianBlockTime(); - if (medianBlockTime == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BTC_NETWORK_ISSUE); + // Does supplied public key match AT creator's public key? + if (!Arrays.equals(creatorPublicKey, atData.getCreatorPublicKey())) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - long now = NTP.getTime(); - - // Check P2SH is funded - Coin p2shBalance = BTC.getInstance().getBalance(p2shAddress.toString()); - if (p2shBalance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - - if (p2shBalance.value < crossChainTradeData.expectedBitcoin) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BTC_BALANCE_ISSUE); - - List fundingOutputs = BTC.getInstance().getUnspentOutputs(p2shAddress.toString()); - if (fundingOutputs.isEmpty()) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - boolean canRedeem = now >= medianBlockTime * 1000L; - if (!canRedeem) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BTC_TOO_SOON); - - Coin redeemAmount = p2shBalance.subtract(Coin.valueOf(redeemRequest.bitcoinMinerFee.unscaledValue().longValue())); + // Good to make MESSAGE - org.bitcoinj.core.Transaction redeemTransaction = BTCACCT.buildRedeemTransaction(redeemAmount, redeemKey, fundingOutputs, redeemScriptBytes, redeemRequest.secret); - boolean wasBroadcast = BTC.getInstance().broadcastTransaction(redeemTransaction); + String atCreatorAddress = Crypto.toAddress(creatorPublicKey); + byte[] messageData = acct.buildCancelMessage(atCreatorAddress); - if (!wasBroadcast) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BTC_NETWORK_ISSUE); + byte[] messageTransactionBytes = buildAtMessage(repository, creatorPublicKey, cancelRequest.atAddress, messageData); - return redeemTransaction.getTxId().toString(); + return Base58.encode(messageTransactionBytes); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } } - private ATData fetchAtDataWithChecking(Repository repository, byte[] creatorPublicKey, String atAddress) throws DataException { + private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atAddress); if (atData == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - // Does supplied public key match that of AT? - if (creatorPublicKey != null && !Arrays.equals(creatorPublicKey, atData.getCreatorPublicKey())) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - // Must be correct AT - check functionality using code hash - if (!Arrays.equals(atData.getCodeHash(), BTCACCT.CODE_BYTES_HASH)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - // No point sending message to AT that's finished if (atData.getIsFinished()) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); @@ -738,27 +444,36 @@ private ATData fetchAtDataWithChecking(Repository repository, byte[] creatorPubl } private byte[] buildAtMessage(Repository repository, byte[] senderPublicKey, String atAddress, byte[] messageData) throws DataException { - PublicKeyAccount creatorAccount = new PublicKeyAccount(repository, senderPublicKey); - long txTimestamp = NTP.getTime(); - byte[] lastReference = creatorAccount.getLastReference(); - if (lastReference == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_REFERENCE); + // senderPublicKey could be ephemeral trade public key where there is no corresponding account and hence no reference + String senderAddress = Crypto.toAddress(senderPublicKey); + byte[] lastReference = repository.getAccountRepository().getLastReference(senderAddress); + final boolean requiresPoW = lastReference == null; + + if (requiresPoW) { + Random random = new Random(); + lastReference = new byte[Transformer.SIGNATURE_LENGTH]; + random.nextBytes(lastReference); + } int version = 4; int nonce = 0; long amount = 0L; Long assetId = null; // no assetId as amount is zero - Long fee = null; + Long fee = 0L; BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, senderPublicKey, fee, null); TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, atAddress, amount, assetId, messageData, false, false); MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); - fee = messageTransaction.calcRecommendedFee(); - messageTransactionData.setFee(fee); + if (requiresPoW) { + messageTransaction.computeNonce(); + } else { + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + } ValidationResult result = messageTransaction.isValidUnconfirmed(); if (result != ValidationResult.OK) @@ -771,4 +486,4 @@ private byte[] buildAtMessage(Repository repository, byte[] senderPublicKey, Str } } -} \ No newline at end of file +} diff --git a/src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java b/src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java new file mode 100644 index 000000000..35a678f25 --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java @@ -0,0 +1,298 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; + +import java.util.List; +import java.util.stream.Collectors; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + +import org.qortal.account.Account; +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.api.model.crosschain.TradeBotRespondRequest; +import org.qortal.asset.Asset; +import org.qortal.controller.Controller; +import org.qortal.controller.tradebot.AcctTradeBot; +import org.qortal.controller.tradebot.TradeBot; +import org.qortal.crosschain.ForeignBlockchain; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.AcctMode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.Base58; + +@Path("/crosschain/tradebot") +@Tag(name = "Cross-Chain (Trade-Bot)") +public class CrossChainTradeBotResource { + + @Context + HttpServletRequest request; + + @GET + @Operation( + summary = "List current trade-bot states", + responses = { + @ApiResponse( + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = TradeBotData.class + ) + ) + ) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public List getTradeBotStates( + @HeaderParam(Security.API_KEY_HEADER) String apiKey, + @Parameter( + description = "Limit to specific blockchain", + example = "LITECOIN", + schema = @Schema(implementation = SupportedBlockchain.class) + ) @QueryParam("foreignBlockchain") SupportedBlockchain foreignBlockchain) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + + if (foreignBlockchain == null) + return allTradeBotData; + + return allTradeBotData.stream().filter(tradeBotData -> tradeBotData.getForeignBlockchain().equals(foreignBlockchain.name())).collect(Collectors.toList()); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST + @Path("/create") + @Operation( + summary = "Create a trade offer (trade-bot entry)", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = TradeBotCreateRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.INSUFFICIENT_BALANCE, ApiError.REPOSITORY_ISSUE, ApiError.ORDER_SIZE_TOO_SMALL}) + @SuppressWarnings("deprecation") + @SecurityRequirement(name = "apiKey") + public String tradeBotCreator(@HeaderParam(Security.API_KEY_HEADER) String apiKey, TradeBotCreateRequest tradeBotCreateRequest) { + Security.checkApiCallAllowed(request); + + if (tradeBotCreateRequest.foreignBlockchain == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + ForeignBlockchain foreignBlockchain = tradeBotCreateRequest.foreignBlockchain.getInstance(); + + // We prefer foreignAmount to deprecated bitcoinAmount + if (tradeBotCreateRequest.foreignAmount == null) + tradeBotCreateRequest.foreignAmount = tradeBotCreateRequest.bitcoinAmount; + + if (!foreignBlockchain.isValidAddress(tradeBotCreateRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (tradeBotCreateRequest.tradeTimeout < 60) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + if (tradeBotCreateRequest.foreignAmount == null || tradeBotCreateRequest.foreignAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ORDER_SIZE_TOO_SMALL); + + if (tradeBotCreateRequest.foreignAmount < foreignBlockchain.getMinimumOrderAmount()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ORDER_SIZE_TOO_SMALL); + + if (tradeBotCreateRequest.qortAmount <= 0 || tradeBotCreateRequest.fundingQortAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ORDER_SIZE_TOO_SMALL); + + if (!Controller.getInstance().isUpToDate()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC); + + try (final Repository repository = RepositoryManager.getRepository()) { + // Do some simple checking first + Account creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); + + if (creator.getConfirmedBalance(Asset.QORT) < tradeBotCreateRequest.fundingQortAmount) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INSUFFICIENT_BALANCE); + + byte[] unsignedBytes = TradeBot.getInstance().createTrade(repository, tradeBotCreateRequest); + if (unsignedBytes == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + return Base58.encode(unsignedBytes); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST + @Path("/respond") + @Operation( + summary = "Respond to a trade offer. NOTE: WILL SPEND FUNDS!)", + description = "Start a new trade-bot entry to respond to chosen trade offer.", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = TradeBotRespondRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE, ApiError.REPOSITORY_ISSUE}) + @SuppressWarnings("deprecation") + @SecurityRequirement(name = "apiKey") + public String tradeBotResponder(@HeaderParam(Security.API_KEY_HEADER) String apiKey, TradeBotRespondRequest tradeBotRespondRequest) { + Security.checkApiCallAllowed(request); + + final String atAddress = tradeBotRespondRequest.atAddress; + + // We prefer foreignKey to deprecated xprv58 + if (tradeBotRespondRequest.foreignKey == null) + tradeBotRespondRequest.foreignKey = tradeBotRespondRequest.xprv58; + + if (tradeBotRespondRequest.foreignKey == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + if (atAddress == null || !Crypto.isValidAtAddress(atAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (tradeBotRespondRequest.receivingAddress == null || !Crypto.isValidAddress(tradeBotRespondRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (!Controller.getInstance().isUpToDate()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC); + + // Extract data from cross-chain trading AT + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = fetchAtDataWithChecking(repository, atAddress); + + // TradeBot uses AT's code hash to map to ACCT + ACCT acct = TradeBot.getInstance().getAcctUsingAtData(atData); + if (acct == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (!acct.getBlockchain().isValidWalletKey(tradeBotRespondRequest.foreignKey)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + if (crossChainTradeData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (crossChainTradeData.mode != AcctMode.OFFERING) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + AcctTradeBot.ResponseResult result = TradeBot.getInstance().startResponse(repository, atData, acct, crossChainTradeData, + tradeBotRespondRequest.foreignKey, tradeBotRespondRequest.receivingAddress); + + switch (result) { + case OK: + return "true"; + + case BALANCE_ISSUE: + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE); + + case NETWORK_ISSUE: + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + + default: + return "false"; + } + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @DELETE + @Operation( + summary = "Delete completed trade", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + example = "93MB2qRDNVLxbmmPuYpLdAqn3u2x9ZhaVZK5wELHueP8" + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String tradeBotDelete(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String tradePrivateKey58) { + Security.checkApiCallAllowed(request); + + final byte[] tradePrivateKey; + try { + tradePrivateKey = Base58.decode(tradePrivateKey58); + + if (tradePrivateKey.length != 32) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + } catch (NumberFormatException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + // Handed off to TradeBot + return TradeBot.getInstance().deleteEntry(repository, tradePrivateKey) ? "true" : "false"; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + // No point sending message to AT that's finished + if (atData.getIsFinished()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + return atData; + } + +} diff --git a/src/main/java/org/qortal/api/resource/ListsResource.java b/src/main/java/org/qortal/api/resource/ListsResource.java new file mode 100644 index 000000000..e0f558df9 --- /dev/null +++ b/src/main/java/org/qortal/api/resource/ListsResource.java @@ -0,0 +1,176 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; + +import org.qortal.api.*; +import org.qortal.api.model.ListRequest; +import org.qortal.crypto.Crypto; +import org.qortal.data.account.AccountData; +import org.qortal.list.ResourceListManager; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + + +@Path("/lists") +@Tag(name = "Lists") +public class ListsResource { + + @Context + HttpServletRequest request; + + + @POST + @Path("/{listName}") + @Operation( + summary = "Add items to a new or existing list", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = ListRequest.class + ) + ) + ), + responses = { + @ApiResponse( + description = "Returns true if all items were processed, false if any couldn't be " + + "processed, or an exception on failure. If false or an exception is returned, " + + "the list will not be updated, and the request will need to be re-issued.", + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String addItemstoList(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("listName") String listName, + ListRequest listRequest) { + Security.checkApiCallAllowed(request); + + if (listName == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + if (listRequest == null || listRequest.items == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + int successCount = 0; + int errorCount = 0; + + for (String item : listRequest.items) { + + boolean success = ResourceListManager.getInstance().addToList(listName, item, false); + if (success) { + successCount++; + } + else { + errorCount++; + } + } + + if (successCount > 0 && errorCount == 0) { + // All were successful, so save the list + ResourceListManager.getInstance().saveList(listName); + return "true"; + } + else { + // Something went wrong, so revert + ResourceListManager.getInstance().revertList(listName); + return "false"; + } + } + + @DELETE + @Path("/{listName}") + @Operation( + summary = "Remove one or more items from a list", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = ListRequest.class + ) + ) + ), + responses = { + @ApiResponse( + description = "Returns true if all items were processed, false if any couldn't be " + + "processed, or an exception on failure. If false or an exception is returned, " + + "the list will not be updated, and the request will need to be re-issued.", + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String removeItemsFromList(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("listName") String listName, + ListRequest listRequest) { + Security.checkApiCallAllowed(request); + + if (listRequest == null || listRequest.items == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + int successCount = 0; + int errorCount = 0; + + for (String address : listRequest.items) { + + // Attempt to remove the item + // Don't save as we will do this at the end of the process + boolean success = ResourceListManager.getInstance().removeFromList(listName, address, false); + if (success) { + successCount++; + } + else { + errorCount++; + } + } + + if (successCount > 0 && errorCount == 0) { + // All were successful, so save the list + ResourceListManager.getInstance().saveList(listName); + return "true"; + } + else { + // Something went wrong, so revert + ResourceListManager.getInstance().revertList(listName); + return "false"; + } + } + + @GET + @Path("/{listName}") + @Operation( + summary = "Fetch all items in a list", + responses = { + @ApiResponse( + description = "A JSON array of items", + content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = String.class))) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String getItemsInList(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("listName") String listName) { + Security.checkApiCallAllowed(request); + return ResourceListManager.getInstance().getJSONStringForList(listName); + } + +} diff --git a/src/main/java/org/qortal/api/resource/PeersResource.java b/src/main/java/org/qortal/api/resource/PeersResource.java index 80cb5fa5b..384611417 100644 --- a/src/main/java/org/qortal/api/resource/PeersResource.java +++ b/src/main/java/org/qortal/api/resource/PeersResource.java @@ -6,30 +6,34 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import org.qortal.api.ApiError; -import org.qortal.api.ApiErrors; -import org.qortal.api.ApiException; -import org.qortal.api.ApiExceptionFactory; -import org.qortal.api.Security; +import org.qortal.api.*; import org.qortal.api.model.ConnectedPeer; +import org.qortal.api.model.PeersSummary; +import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; +import org.qortal.controller.Synchronizer.SynchronizationResult; +import org.qortal.data.block.BlockSummaryData; import org.qortal.data.network.PeerData; import org.qortal.network.Network; +import org.qortal.network.Peer; import org.qortal.network.PeerAddress; import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; import org.qortal.utils.ExecuteProduceConsume; import org.qortal.utils.NTP; @@ -122,7 +126,8 @@ public List getSelfPeers() { ) } ) - public ExecuteProduceConsume.StatsSnapshot getEngineStats() { + @SecurityRequirement(name = "apiKey") + public ExecuteProduceConsume.StatsSnapshot getEngineStats(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { Security.checkApiCallAllowed(request); return Network.getInstance().getStatsSnapshot(); @@ -159,7 +164,8 @@ public ExecuteProduceConsume.StatsSnapshot getEngineStats() { @ApiErrors({ ApiError.INVALID_NETWORK_ADDRESS, ApiError.REPOSITORY_ISSUE }) - public String addPeer(String address) { + @SecurityRequirement(name = "apiKey") + public String addPeer(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String address) { Security.checkApiCallAllowed(request); final Long addedWhen = NTP.getTime(); @@ -213,7 +219,8 @@ public String addPeer(String address) { @ApiErrors({ ApiError.INVALID_NETWORK_ADDRESS, ApiError.REPOSITORY_ISSUE }) - public String removePeer(String address) { + @SecurityRequirement(name = "apiKey") + public String removePeer(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String address) { Security.checkApiCallAllowed(request); try { @@ -248,7 +255,8 @@ public String removePeer(String address) { @ApiErrors({ ApiError.REPOSITORY_ISSUE }) - public String removeKnownPeers(String address) { + @SecurityRequirement(name = "apiKey") + public String removeKnownPeers(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String address) { Security.checkApiCallAllowed(request); try { @@ -260,4 +268,100 @@ public String removeKnownPeers(String address) { } } + @POST + @Path("/commonblock") + @Operation( + summary = "Report common block with given peer.", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "node2.qortal.org" + ) + ) + ), + responses = { + @ApiResponse( + description = "the block", + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = BlockSummaryData.class + ) + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public List commonBlock(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String targetPeerAddress) { + Security.checkApiCallAllowed(request); + + try { + // Try to resolve passed address to make things easier + PeerAddress peerAddress = PeerAddress.fromString(targetPeerAddress); + InetSocketAddress resolvedAddress = peerAddress.toSocketAddress(); + + List peers = Network.getInstance().getHandshakedPeers(); + Peer targetPeer = peers.stream().filter(peer -> peer.getResolvedAddress().equals(resolvedAddress)).findFirst().orElse(null); + + if (targetPeer == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + try (final Repository repository = RepositoryManager.getRepository()) { + int ourInitialHeight = Controller.getInstance().getChainHeight(); + boolean force = true; + List peerBlockSummaries = new ArrayList<>(); + + SynchronizationResult findCommonBlockResult = Synchronizer.getInstance().fetchSummariesFromCommonBlock(repository, targetPeer, ourInitialHeight, force, peerBlockSummaries, true); + if (findCommonBlockResult != SynchronizationResult.OK) + return null; + + return peerBlockSummaries; + } + } catch (IllegalArgumentException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } catch (UnknownHostException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (InterruptedException e) { + return null; + } + } + + @GET + @Path("/summary") + @Operation( + summary = "Returns total inbound and outbound connections for connected peers", + responses = { + @ApiResponse( + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + array = @ArraySchema( + schema = @Schema( + implementation = PeersSummary.class + ) + ) + ) + ) + } + ) + public PeersSummary peersSummary() { + PeersSummary peersSummary = new PeersSummary(); + + List connectedPeers = Network.getInstance().getConnectedPeers().stream().collect(Collectors.toList()); + for (Peer peer : connectedPeers) { + if (!peer.isOutbound()) { + peersSummary.inboundConnections++; + } + else { + peersSummary.outboundConnections++; + } + } + return peersSummary; + } + } diff --git a/src/main/java/org/qortal/api/resource/RenderResource.java b/src/main/java/org/qortal/api/resource/RenderResource.java new file mode 100644 index 000000000..97411e54b --- /dev/null +++ b/src/main/java/org/qortal/api/resource/RenderResource.java @@ -0,0 +1,200 @@ +package org.qortal.api.resource; + +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import java.io.*; +import java.nio.file.Paths; +import java.util.Map; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.api.ApiError; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.arbitrary.misc.Service; +import org.qortal.arbitrary.*; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.controller.arbitrary.ArbitraryDataRenderManager; +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.repository.DataException; +import org.qortal.settings.Settings; +import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.utils.Base58; + + +@Path("/render") +@Tag(name = "Render") +public class RenderResource { + + private static final Logger LOGGER = LogManager.getLogger(RenderResource.class); + + @Context HttpServletRequest request; + @Context HttpServletResponse response; + @Context ServletContext context; + + @POST + @Path("/preview") + @Operation( + summary = "Generate preview URL based on a user-supplied path and service", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "/Users/user/Documents/MyStaticWebsite" + ) + ) + ), + responses = { + @ApiResponse( + description = "a temporary URL to preview the website", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String preview(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String directoryPath) { + Security.checkApiCallAllowed(request); + Method method = Method.PUT; + Compression compression = Compression.ZIP; + + ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), null, Service.WEBSITE, null, method, compression); + try { + arbitraryDataWriter.save(); + } catch (IOException | DataException | InterruptedException | MissingDataException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); + } catch (RuntimeException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + + ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile(); + if (arbitraryDataFile != null) { + String digest58 = arbitraryDataFile.digest58(); + if (digest58 != null) { + return "http://localhost:12393/render/hash/" + digest58 + "?secret=" + Base58.encode(arbitraryDataFile.getSecret()); + } + } + return "Unable to generate preview URL"; + } + + @POST + @Path("/authorize/{resourceId}") + @SecurityRequirement(name = "apiKey") + public boolean authorizeResource(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("resourceId") String resourceId) { + Security.checkApiCallAllowed(request); + Security.disallowLoopbackRequestsIfAuthBypassEnabled(request); + ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, null, null); + ArbitraryDataRenderManager.getInstance().addToAuthorizedResources(resource); + return true; + } + + @POST + @Path("authorize/{service}/{resourceId}") + @SecurityRequirement(name = "apiKey") + public boolean authorizeResource(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") Service service, + @PathParam("resourceId") String resourceId) { + Security.checkApiCallAllowed(request); + Security.disallowLoopbackRequestsIfAuthBypassEnabled(request); + ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, null); + ArbitraryDataRenderManager.getInstance().addToAuthorizedResources(resource); + return true; + } + + @POST + @Path("authorize/{service}/{resourceId}/{identifier}") + @SecurityRequirement(name = "apiKey") + public boolean authorizeResource(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") Service service, + @PathParam("resourceId") String resourceId, + @PathParam("identifier") String identifier) { + Security.checkApiCallAllowed(request); + Security.disallowLoopbackRequestsIfAuthBypassEnabled(request); + ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, identifier); + ArbitraryDataRenderManager.getInstance().addToAuthorizedResources(resource); + return true; + } + + @GET + @Path("/signature/{signature}") + @SecurityRequirement(name = "apiKey") + public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature) { + Security.requirePriorAuthorization(request, signature, Service.WEBSITE, null); + return this.get(signature, ResourceIdType.SIGNATURE, null, "/", null, "/render/signature", true, true); + } + + @GET + @Path("/signature/{signature}/{path:.*}") + @SecurityRequirement(name = "apiKey") + public HttpServletResponse getPathBySignature(@PathParam("signature") String signature, @PathParam("path") String inPath) { + Security.requirePriorAuthorization(request, signature, Service.WEBSITE, null); + return this.get(signature, ResourceIdType.SIGNATURE, null, inPath,null, "/render/signature", true, true); + } + + @GET + @Path("/hash/{hash}") + @SecurityRequirement(name = "apiKey") + public HttpServletResponse getIndexByHash(@PathParam("hash") String hash58, @QueryParam("secret") String secret58) { + Security.requirePriorAuthorization(request, hash58, Service.WEBSITE, null); + return this.get(hash58, ResourceIdType.FILE_HASH, Service.WEBSITE, "/", secret58, "/render/hash", true, false); + } + + @GET + @Path("/hash/{hash}/{path:.*}") + @SecurityRequirement(name = "apiKey") + public HttpServletResponse getPathByHash(@PathParam("hash") String hash58, @PathParam("path") String inPath, + @QueryParam("secret") String secret58) { + Security.requirePriorAuthorization(request, hash58, Service.WEBSITE, null); + return this.get(hash58, ResourceIdType.FILE_HASH, Service.WEBSITE, inPath, secret58, "/render/hash", true, false); + } + + @GET + @Path("{service}/{name}/{path:.*}") + @SecurityRequirement(name = "apiKey") + public HttpServletResponse getPathByName(@PathParam("service") Service service, + @PathParam("name") String name, + @PathParam("path") String inPath) { + Security.requirePriorAuthorization(request, name, service, null); + String prefix = String.format("/render/%s", service); + return this.get(name, ResourceIdType.NAME, service, inPath, null, prefix, true, true); + } + + @GET + @Path("{service}/{name}") + @SecurityRequirement(name = "apiKey") + public HttpServletResponse getIndexByName(@PathParam("service") Service service, + @PathParam("name") String name) { + Security.requirePriorAuthorization(request, name, service, null); + String prefix = String.format("/render/%s", service); + return this.get(name, ResourceIdType.NAME, service, "/", null, prefix, true, true); + } + + + + private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String inPath, + String secret58, String prefix, boolean usePrefix, boolean async) { + + ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, inPath, + secret58, prefix, usePrefix, async, request, response, context); + return renderer.render(); + } + +} diff --git a/src/main/java/org/qortal/api/resource/TransactionsResource.java b/src/main/java/org/qortal/api/resource/TransactionsResource.java index 77218a696..9bc6d497e 100644 --- a/src/main/java/org/qortal/api/resource/TransactionsResource.java +++ b/src/main/java/org/qortal/api/resource/TransactionsResource.java @@ -9,6 +9,8 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -44,6 +46,7 @@ import org.qortal.utils.Base58; import com.google.common.primitives.Bytes; +import org.qortal.utils.NTP; @Path("/transactions") @Tag(name = "Transactions") @@ -348,7 +351,7 @@ public List searchTransactions(@QueryParam("startBlock") Intege try (final Repository repository = RepositoryManager.getRepository()) { List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(startBlock, blockLimit, txGroupId, - txTypes, null, address, confirmationStatus, limit, offset, reverse); + txTypes, null, null, address, confirmationStatus, limit, offset, reverse); // Expand signatures to transactions List transactions = new ArrayList<>(signatures.size()); @@ -363,6 +366,83 @@ public List searchTransactions(@QueryParam("startBlock") Intege } } + @GET + @Path("/unitfee") + @Operation( + summary = "Get transaction unit fee", + responses = { + @ApiResponse( + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "number" + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE + }) + public long getTransactionUnitFee(@QueryParam("txType") TransactionType txType, + @QueryParam("timestamp") Long timestamp, + @QueryParam("level") Integer accountLevel) { + try { + if (timestamp == null) { + timestamp = NTP.getTime(); + } + + Constructor constructor = txType.constructor; + Transaction transaction = (Transaction) constructor.newInstance(null, null); + // FUTURE: add accountLevel parameter to transaction.getUnitFee() if needed + return transaction.getUnitFee(timestamp); + + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e); + } + } + + @POST + @Path("/fee") + @Operation( + summary = "Get recommended fee for supplied transaction data", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + ) + @ApiErrors({ + ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE + }) + public long getRecommendedTransactionFee(String rawInputBytes58) { + byte[] rawInputBytes = Base58.decode(rawInputBytes58); + if (rawInputBytes.length == 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.JSON); + + try (final Repository repository = RepositoryManager.getRepository()) { + + // Append null signature on the end before transformation + byte[] rawBytes = Bytes.concat(rawInputBytes, new byte[TransactionTransformer.SIGNATURE_LENGTH]); + + TransactionData transactionData = TransactionTransformer.fromBytes(rawBytes); + if (transactionData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + Transaction transaction = Transaction.fromData(repository, transactionData); + return transaction.calcRecommendedFee(); + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + } + } + @GET @Path("/creator/{publickey}") @Operation( @@ -418,32 +498,83 @@ public List findCreatorsTransactions(@PathParam("publickey") St } @POST - @Path("/sign") + @Path("/convert") @Operation( - summary = "Sign a raw, unsigned transaction", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = SimpleTransactionSignRequest.class - ) - ) - ), - responses = { - @ApiResponse( - description = "raw, signed transaction encoded in Base58", - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string" + summary = "Convert transaction bytes into bytes for signing", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + description = "raw, unsigned transaction in base58 encoding", + example = "raw transaction base58" + ) ) - ) - ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned transaction encoded in Base58, ready for signing", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.NON_PRODUCTION, ApiError.TRANSFORMATION_ERROR + }) + public String convertTransactionForSigning(String rawInputBytes58) { + byte[] rawInputBytes = Base58.decode(rawInputBytes58); + if (rawInputBytes.length == 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.JSON); + + try { + // Append null signature on the end before transformation + byte[] rawBytes = Bytes.concat(rawInputBytes, new byte[TransactionTransformer.SIGNATURE_LENGTH]); + + TransactionData transactionData = TransactionTransformer.fromBytes(rawBytes); + if (transactionData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + byte[] convertedBytes = TransactionTransformer.toBytesForSigning(transactionData); + + return Base58.encode(convertedBytes); + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); } + } + + @POST + @Path("/sign") + @Operation( + summary = "Sign a raw, unsigned transaction", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = SimpleTransactionSignRequest.class + ) + ) + ), + responses = { + @ApiResponse( + description = "raw, signed transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } ) @ApiErrors({ - ApiError.NON_PRODUCTION, ApiError.INVALID_PRIVATE_KEY, ApiError.TRANSFORMATION_ERROR + ApiError.NON_PRODUCTION, ApiError.INVALID_PRIVATE_KEY, ApiError.TRANSFORMATION_ERROR }) public String signTransaction(SimpleTransactionSignRequest signRequest) { if (Settings.getInstance().isApiRestricted()) @@ -510,14 +641,19 @@ public String processTransaction(String rawBytes58) { if (!Controller.getInstance().isUpToDate()) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC); - try (final Repository repository = RepositoryManager.getRepository()) { - byte[] rawBytes = Base58.decode(rawBytes58); + byte[] rawBytes = Base58.decode(rawBytes58); - TransactionData transactionData = TransactionTransformer.fromBytes(rawBytes); + TransactionData transactionData; + try { + transactionData = TransactionTransformer.fromBytes(rawBytes); + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + } - if (transactionData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + if (transactionData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + try (final Repository repository = RepositoryManager.getRepository()) { Transaction transaction = Transaction.fromData(repository, transactionData); if (!transaction.isSignatureValid()) @@ -535,16 +671,9 @@ public String processTransaction(String rawBytes58) { blockchainLock.unlock(); } - // Notify controller of new transaction - Controller.getInstance().onNewTransaction(transactionData, null); - return "true"; } catch (NumberFormatException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); - } catch (TransformationException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); - } catch (ApiException e) { - throw e; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } catch (InterruptedException e) { diff --git a/src/main/java/org/qortal/api/resource/UtilsResource.java b/src/main/java/org/qortal/api/resource/UtilsResource.java index cc492a2f3..54ea660b5 100644 --- a/src/main/java/org/qortal/api/resource/UtilsResource.java +++ b/src/main/java/org/qortal/api/resource/UtilsResource.java @@ -33,7 +33,6 @@ import org.qortal.transform.Transformer; import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.transform.transaction.TransactionTransformer.Transformation; -import org.qortal.utils.BIP39; import org.qortal.utils.Base58; import com.google.common.hash.HashCode; @@ -195,123 +194,6 @@ public String random(@QueryParam("length") Integer length) { return Base58.encode(random); } - @GET - @Path("/mnemonic") - @Operation( - summary = "Generate 12-word BIP39 mnemonic", - description = "Optionally pass 16-byte, base58-encoded entropy or entropy will be internally generated.
" - + "Example entropy input: YcVfxkQb6JRzqk5kF2tNLv", - responses = { - @ApiResponse( - description = "mnemonic", - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string" - ) - ) - ) - } - ) - @ApiErrors({ApiError.NON_PRODUCTION, ApiError.INVALID_DATA}) - public String getMnemonic(@QueryParam("entropy") String suppliedEntropy) { - if (Settings.getInstance().isApiRestricted()) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); - - /* - * BIP39 word lists have 2048 entries so can be represented by 11 bits. - * UUID (128bits) and another 4 bits gives 132 bits. - * 132 bits, divided by 11, gives 12 words. - */ - byte[] entropy; - if (suppliedEntropy != null) { - // Use caller-supplied entropy input - try { - entropy = Base58.decode(suppliedEntropy); - } catch (NumberFormatException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - - // Must be 16-bytes - if (entropy.length != 16) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } else { - // Generate entropy internally - UUID uuid = UUID.randomUUID(); - - byte[] uuidMSB = Longs.toByteArray(uuid.getMostSignificantBits()); - byte[] uuidLSB = Longs.toByteArray(uuid.getLeastSignificantBits()); - entropy = Bytes.concat(uuidMSB, uuidLSB); - } - - // Use SHA256 to generate more bits - byte[] hash = Crypto.digest(entropy); - - // Append first 4 bits from hash to end. (Actually 8 bits but we only use 4). - byte checksum = (byte) (hash[0] & 0xf0); - entropy = Bytes.concat(entropy, new byte[] { - checksum - }); - - return BIP39.encode(entropy, "en"); - } - - @POST - @Path("/mnemonic") - @Operation( - summary = "Calculate binary entropy from 12-word BIP39 mnemonic", - description = "Returns the base58-encoded binary form, or \"false\" if mnemonic is invalid.", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string" - ) - ) - ), - responses = { - @ApiResponse( - description = "entropy in base58", - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string" - ) - ) - ) - } - ) - @ApiErrors({ApiError.NON_PRODUCTION}) - public String fromMnemonic(String mnemonic) { - if (Settings.getInstance().isApiRestricted()) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); - - if (mnemonic.isEmpty()) - return "false"; - - // Strip leading/trailing whitespace if any - mnemonic = mnemonic.trim(); - - String[] phraseWords = mnemonic.split(" "); - if (phraseWords.length != 12) - return "false"; - - // Convert BIP39 mnemonic to binary - byte[] binary = BIP39.decode(phraseWords, "en"); - if (binary == null) - return "false"; - - byte[] entropy = Arrays.copyOf(binary, 16); // 132 bits is 16.5 bytes, but we're discarding checksum nybble - - byte checksumNybble = (byte) (binary[16] & 0xf0); - byte[] checksum = Crypto.digest(entropy); - if (checksumNybble != (byte) (checksum[0] & 0xf0)) - return "false"; - - return Base58.encode(entropy); - } - @POST @Path("/privatekey") @Operation( diff --git a/src/main/java/org/qortal/api/websocket/ActiveChatsWebSocket.java b/src/main/java/org/qortal/api/websocket/ActiveChatsWebSocket.java index b85b7891e..405fe7e54 100644 --- a/src/main/java/org/qortal/api/websocket/ActiveChatsWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/ActiveChatsWebSocket.java @@ -6,11 +6,12 @@ import java.util.concurrent.atomic.AtomicReference; import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketException; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; import org.eclipse.jetty.websocket.api.annotations.WebSocket; -import org.eclipse.jetty.websocket.servlet.WebSocketServlet; import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; import org.qortal.controller.ChatNotifier; import org.qortal.crypto.Crypto; @@ -22,7 +23,7 @@ @WebSocket @SuppressWarnings("serial") -public class ActiveChatsWebSocket extends WebSocketServlet implements ApiWebSocket { +public class ActiveChatsWebSocket extends ApiWebSocket { @Override public void configure(WebSocketServletFactory factory) { @@ -30,8 +31,9 @@ public void configure(WebSocketServletFactory factory) { } @OnWebSocketConnect + @Override public void onWebSocketConnect(Session session) { - Map pathParams = this.getPathParams(session, "/{address}"); + Map pathParams = getPathParams(session, "/{address}"); String address = pathParams.get("address"); if (address == null || !Crypto.isValidAddress(address)) { @@ -48,12 +50,19 @@ public void onWebSocketConnect(Session session) { } @OnWebSocketClose + @Override public void onWebSocketClose(Session session, int statusCode, String reason) { ChatNotifier.getInstance().deregister(session); } + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* ignored */ + } + @OnWebSocketMessage public void onWebSocketMessage(Session session, String message) { + /* ignored */ } private void onNotify(Session session, ChatTransactionData chatTransactionData, String ourAddress, AtomicReference previousOutput) { @@ -70,7 +79,7 @@ private void onNotify(Session session, ChatTransactionData chatTransactionData, StringWriter stringWriter = new StringWriter(); - this.marshall(stringWriter, activeChats); + marshall(stringWriter, activeChats); // Only output if something has changed String output = stringWriter.toString(); @@ -78,8 +87,8 @@ private void onNotify(Session session, ChatTransactionData chatTransactionData, return; previousOutput.set(output); - session.getRemote().sendString(output); - } catch (DataException | IOException e) { + session.getRemote().sendStringByFuture(output); + } catch (DataException | IOException | WebSocketException e) { // No output this time? } } diff --git a/src/main/java/org/qortal/api/websocket/AdminStatusWebSocket.java b/src/main/java/org/qortal/api/websocket/AdminStatusWebSocket.java index 2a957921e..6556d1407 100644 --- a/src/main/java/org/qortal/api/websocket/AdminStatusWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/AdminStatusWebSocket.java @@ -5,62 +5,95 @@ import java.util.concurrent.atomic.AtomicReference; import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketException; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; import org.eclipse.jetty.websocket.api.annotations.WebSocket; -import org.eclipse.jetty.websocket.servlet.WebSocketServlet; import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; import org.qortal.api.model.NodeStatus; -import org.qortal.controller.StatusNotifier; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; +import org.qortal.controller.Controller; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.Listener; @WebSocket @SuppressWarnings("serial") -public class AdminStatusWebSocket extends WebSocketServlet implements ApiWebSocket { +public class AdminStatusWebSocket extends ApiWebSocket implements Listener { + + private static final AtomicReference previousOutput = new AtomicReference<>(null); @Override public void configure(WebSocketServletFactory factory) { factory.register(AdminStatusWebSocket.class); + + try { + previousOutput.set(buildStatusString()); + } catch (IOException e) { + // How to fail properly? + return; + } + + EventBus.INSTANCE.addListener(this::listen); + } + + @Override + public void listen(Event event) { + if (!(event instanceof Controller.StatusChangeEvent)) + return; + + String newOutput; + try { + newOutput = buildStatusString(); + } catch (IOException e) { + // Ignore this time? + return; + } + + if (previousOutput.getAndUpdate(currentValue -> newOutput).equals(newOutput)) + // Output hasn't changed, so don't send anything + return; + + for (Session session : getSessions()) + this.sendStatus(session, newOutput); } @OnWebSocketConnect + @Override public void onWebSocketConnect(Session session) { - AtomicReference previousOutput = new AtomicReference<>(null); - - StatusNotifier.Listener listener = timestamp -> onNotify(session, previousOutput); - StatusNotifier.getInstance().register(session, listener); + this.sendStatus(session, previousOutput.get()); - this.onNotify(session, previousOutput); + super.onWebSocketConnect(session); } @OnWebSocketClose + @Override public void onWebSocketClose(Session session, int statusCode, String reason) { - StatusNotifier.getInstance().deregister(session); + super.onWebSocketClose(session, statusCode, reason); + } + + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* We ignore errors for now, but method here to silence log spam */ } @OnWebSocketMessage public void onWebSocketMessage(Session session, String message) { + /* ignored */ } - private void onNotify(Session session,AtomicReference previousOutput) { - try (final Repository repository = RepositoryManager.getRepository()) { - NodeStatus nodeStatus = new NodeStatus(); - - StringWriter stringWriter = new StringWriter(); - - this.marshall(stringWriter, nodeStatus); - - // Only output if something has changed - String output = stringWriter.toString(); - if (output.equals(previousOutput.get())) - return; + private static String buildStatusString() throws IOException { + NodeStatus nodeStatus = new NodeStatus(); + StringWriter stringWriter = new StringWriter(); + marshall(stringWriter, nodeStatus); + return stringWriter.toString(); + } - previousOutput.set(output); - session.getRemote().sendString(output); - } catch (DataException | IOException e) { + private void sendStatus(Session session, String status) { + try { + session.getRemote().sendStringByFuture(status); + } catch (WebSocketException e) { // No output this time? } } diff --git a/src/main/java/org/qortal/api/websocket/ApiWebSocket.java b/src/main/java/org/qortal/api/websocket/ApiWebSocket.java index 9209c5b90..f6a439ea0 100644 --- a/src/main/java/org/qortal/api/websocket/ApiWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/ApiWebSocket.java @@ -3,7 +3,10 @@ import java.io.IOException; import java.io.StringWriter; import java.io.Writer; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.xml.bind.JAXBContext; @@ -13,24 +16,28 @@ import org.eclipse.jetty.http.pathmap.UriTemplatePathSpec; import org.eclipse.jetty.websocket.api.Session; import org.eclipse.jetty.websocket.servlet.ServletUpgradeRequest; +import org.eclipse.jetty.websocket.servlet.WebSocketServlet; import org.eclipse.persistence.jaxb.JAXBContextFactory; import org.eclipse.persistence.jaxb.MarshallerProperties; import org.qortal.api.ApiError; import org.qortal.api.ApiErrorRoot; -interface ApiWebSocket { +@SuppressWarnings("serial") +abstract class ApiWebSocket extends WebSocketServlet { - default String getPathInfo(Session session) { + private static final Map, List> SESSIONS_BY_CLASS = new HashMap<>(); + + protected static String getPathInfo(Session session) { ServletUpgradeRequest upgradeRequest = (ServletUpgradeRequest) session.getUpgradeRequest(); return upgradeRequest.getHttpServletRequest().getPathInfo(); } - default Map getPathParams(Session session, String pathSpec) { + protected static Map getPathParams(Session session, String pathSpec) { UriTemplatePathSpec uriTemplatePathSpec = new UriTemplatePathSpec(pathSpec); - return uriTemplatePathSpec.getPathParams(this.getPathInfo(session)); + return uriTemplatePathSpec.getPathParams(getPathInfo(session)); } - default void sendError(Session session, ApiError apiError) { + protected static void sendError(Session session, ApiError apiError) { ApiErrorRoot apiErrorRoot = new ApiErrorRoot(); apiErrorRoot.setApiError(apiError); @@ -43,7 +50,7 @@ default void sendError(Session session, ApiError apiError) { } } - default void marshall(Writer writer, Object object) throws IOException { + protected static void marshall(Writer writer, Object object) throws IOException { Marshaller marshaller = createMarshaller(object.getClass()); try { @@ -53,7 +60,7 @@ default void marshall(Writer writer, Object object) throws IOException { } } - default void marshall(Writer writer, Collection collection) throws IOException { + protected static void marshall(Writer writer, Collection collection) throws IOException { // If collection is empty then we're returning "[]" anyway if (collection.isEmpty()) { writer.append("[]"); @@ -92,4 +99,24 @@ private static Marshaller createMarshaller(Class objectClass) { } } + public void onWebSocketConnect(Session session) { + synchronized (SESSIONS_BY_CLASS) { + SESSIONS_BY_CLASS.computeIfAbsent(this.getClass(), clazz -> new ArrayList<>()).add(session); + } + } + + public void onWebSocketClose(Session session, int statusCode, String reason) { + synchronized (SESSIONS_BY_CLASS) { + List sessions = SESSIONS_BY_CLASS.get(this.getClass()); + if (sessions != null) + sessions.remove(session); + } + } + + protected List getSessions() { + synchronized (SESSIONS_BY_CLASS) { + return new ArrayList<>(SESSIONS_BY_CLASS.get(this.getClass())); + } + } + } diff --git a/src/main/java/org/qortal/api/websocket/BlocksWebSocket.java b/src/main/java/org/qortal/api/websocket/BlocksWebSocket.java index 398cdd338..20847b7bb 100644 --- a/src/main/java/org/qortal/api/websocket/BlocksWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/BlocksWebSocket.java @@ -2,17 +2,23 @@ import java.io.IOException; import java.io.StringWriter; +import java.util.List; import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketException; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; import org.eclipse.jetty.websocket.api.annotations.WebSocket; -import org.eclipse.jetty.websocket.servlet.WebSocketServlet; import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; import org.qortal.api.ApiError; -import org.qortal.controller.BlockNotifier; +import org.qortal.controller.Controller; import org.qortal.data.block.BlockData; +import org.qortal.data.block.BlockSummaryData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.Listener; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -20,22 +26,42 @@ @WebSocket @SuppressWarnings("serial") -public class BlocksWebSocket extends WebSocketServlet implements ApiWebSocket { +public class BlocksWebSocket extends ApiWebSocket implements Listener { @Override public void configure(WebSocketServletFactory factory) { factory.register(BlocksWebSocket.class); + + EventBus.INSTANCE.addListener(this::listen); + } + + @Override + public void listen(Event event) { + if (!(event instanceof Controller.NewBlockEvent)) + return; + + BlockData blockData = ((Controller.NewBlockEvent) event).getBlockData(); + BlockSummaryData blockSummary = new BlockSummaryData(blockData); + + for (Session session : getSessions()) + sendBlockSummary(session, blockSummary); } @OnWebSocketConnect + @Override public void onWebSocketConnect(Session session) { - BlockNotifier.Listener listener = blockData -> onNotify(session, blockData); - BlockNotifier.getInstance().register(session, listener); + super.onWebSocketConnect(session); } @OnWebSocketClose + @Override public void onWebSocketClose(Session session, int statusCode, String reason) { - BlockNotifier.getInstance().deregister(session); + super.onWebSocketClose(session, statusCode, reason); + } + + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* We ignore errors for now, but method here to silence log spam */ } @OnWebSocketMessage @@ -53,13 +79,19 @@ public void onWebSocketMessage(Session session, String message) { } try (final Repository repository = RepositoryManager.getRepository()) { - BlockData blockData = repository.getBlockRepository().fromSignature(signature); - if (blockData == null) { + int height = repository.getBlockRepository().getHeightFromSignature(signature); + if (height == 0) { + sendError(session, ApiError.BLOCK_UNKNOWN); + return; + } + + List blockSummaries = repository.getBlockRepository().getBlockSummaries(height, height); + if (blockSummaries == null || blockSummaries.isEmpty()) { sendError(session, ApiError.BLOCK_UNKNOWN); return; } - onNotify(session, blockData); + sendBlockSummary(session, blockSummaries.get(0)); } catch (DataException e) { sendError(session, ApiError.REPOSITORY_ISSUE); } @@ -82,26 +114,26 @@ public void onWebSocketMessage(Session session, String message) { } try (final Repository repository = RepositoryManager.getRepository()) { - BlockData blockData = repository.getBlockRepository().fromHeight(height); - if (blockData == null) { + List blockSummaries = repository.getBlockRepository().getBlockSummaries(height, height); + if (blockSummaries == null || blockSummaries.isEmpty()) { sendError(session, ApiError.BLOCK_UNKNOWN); return; } - onNotify(session, blockData); + sendBlockSummary(session, blockSummaries.get(0)); } catch (DataException e) { sendError(session, ApiError.REPOSITORY_ISSUE); } } - private void onNotify(Session session, BlockData blockData) { + private void sendBlockSummary(Session session, BlockSummaryData blockSummary) { StringWriter stringWriter = new StringWriter(); try { - this.marshall(stringWriter, blockData); + marshall(stringWriter, blockSummary); - session.getRemote().sendString(stringWriter.toString()); - } catch (IOException e) { + session.getRemote().sendStringByFuture(stringWriter.toString()); + } catch (IOException | WebSocketException e) { // No output this time } } diff --git a/src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java b/src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java index ef04b9504..3dc2d4949 100644 --- a/src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java @@ -8,11 +8,12 @@ import java.util.Map; import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketException; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; import org.eclipse.jetty.websocket.api.annotations.WebSocket; -import org.eclipse.jetty.websocket.servlet.WebSocketServlet; import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; import org.qortal.controller.ChatNotifier; import org.qortal.data.chat.ChatMessage; @@ -23,7 +24,7 @@ @WebSocket @SuppressWarnings("serial") -public class ChatMessagesWebSocket extends WebSocketServlet implements ApiWebSocket { +public class ChatMessagesWebSocket extends ApiWebSocket { @Override public void configure(WebSocketServletFactory factory) { @@ -31,6 +32,7 @@ public void configure(WebSocketServletFactory factory) { } @OnWebSocketConnect + @Override public void onWebSocketConnect(Session session) { Map> queryParams = session.getUpgradeRequest().getParameterMap(); @@ -85,12 +87,19 @@ public void onWebSocketConnect(Session session) { } @OnWebSocketClose + @Override public void onWebSocketClose(Session session, int statusCode, String reason) { ChatNotifier.getInstance().deregister(session); } + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* ignored */ + } + @OnWebSocketMessage public void onWebSocketMessage(Session session, String message) { + /* ignored */ } private void onNotify(Session session, ChatTransactionData chatTransactionData, int txGroupId) { @@ -106,6 +115,9 @@ private void onNotify(Session session, ChatTransactionData chatTransactionData, } private void onNotify(Session session, ChatTransactionData chatTransactionData, List involvingAddresses) { + if (chatTransactionData == null) + return; + // We only want direct/non-group messages where sender/recipient match our addresses String recipient = chatTransactionData.getRecipient(); if (recipient == null) @@ -123,10 +135,10 @@ private void sendMessages(Session session, List chatMessages) { StringWriter stringWriter = new StringWriter(); try { - this.marshall(stringWriter, chatMessages); + marshall(stringWriter, chatMessages); - session.getRemote().sendString(stringWriter.toString()); - } catch (IOException e) { + session.getRemote().sendStringByFuture(stringWriter.toString()); + } catch (IOException | WebSocketException e) { // No output this time? } } diff --git a/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java b/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java new file mode 100644 index 000000000..26d131c42 --- /dev/null +++ b/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java @@ -0,0 +1,244 @@ +package org.qortal.api.websocket; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.qortal.controller.Controller; +import org.qortal.crypto.Crypto; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.Listener; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.transaction.PresenceTransaction.PresenceType; +import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +@WebSocket +@SuppressWarnings("serial") +public class PresenceWebSocket extends ApiWebSocket implements Listener { + + @XmlAccessorType(XmlAccessType.FIELD) + @SuppressWarnings("unused") + private static class PresenceInfo { + private final PresenceType presenceType; + private final String publicKey; + private final long timestamp; + private final String address; + + protected PresenceInfo() { + this.presenceType = null; + this.publicKey = null; + this.timestamp = 0L; + this.address = null; + } + + public PresenceInfo(PresenceType presenceType, String pubKey58, long timestamp) { + this.presenceType = presenceType; + this.publicKey = pubKey58; + this.timestamp = timestamp; + this.address = Crypto.toAddress(Base58.decode(this.publicKey)); + } + + public PresenceType getPresenceType() { + return this.presenceType; + } + + public String getPublicKey() { + return this.publicKey; + } + + public long getTimestamp() { + return this.timestamp; + } + + public String getAddress() { + return this.address; + } + } + + /** Outer map key is PresenceType (enum), inner map key is public key in base58, inner map value is timestamp */ + private static final Map> currentEntries = Collections.synchronizedMap(new EnumMap<>(PresenceType.class)); + + /** (Optional) PresenceType used for filtering by that Session. */ + private static final Map sessionPresenceTypes = Collections.synchronizedMap(new HashMap<>()); + + @Override + public void configure(WebSocketServletFactory factory) { + factory.register(PresenceWebSocket.class); + + try (final Repository repository = RepositoryManager.getRepository()) { + populateCurrentInfo(repository); + } catch (DataException e) { + // How to fail properly? + return; + } + + EventBus.INSTANCE.addListener(this::listen); + } + + @Override + public void listen(Event event) { + // We use NewBlockEvent as a proxy for 1-minute timer + if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Controller.NewBlockEvent)) + return; + + removeOldEntries(); + + if (event instanceof Controller.NewBlockEvent) + // We only wanted a chance to cull old entries + return; + + TransactionData transactionData = ((Controller.NewTransactionEvent) event).getTransactionData(); + + if (transactionData.getType() != TransactionType.PRESENCE) + return; + + PresenceTransactionData presenceData = (PresenceTransactionData) transactionData; + PresenceType presenceType = presenceData.getPresenceType(); + + // Put/replace for this publickey making sure we keep newest timestamp + String pubKey58 = Base58.encode(presenceData.getCreatorPublicKey()); + long ourTimestamp = presenceData.getTimestamp(); + long computedTimestamp = mergePresence(presenceType, pubKey58, ourTimestamp); + + if (computedTimestamp != ourTimestamp) + // nothing changed + return; + + List presenceInfo = Collections.singletonList(new PresenceInfo(presenceType, pubKey58, computedTimestamp)); + + // Notify sessions + for (Session session : getSessions()) { + PresenceType sessionPresenceType = sessionPresenceTypes.get(session); + + if (sessionPresenceType == null || sessionPresenceType == presenceType) + sendPresenceInfo(session, presenceInfo); + } + } + + @OnWebSocketConnect + @Override + public void onWebSocketConnect(Session session) { + Map> queryParams = session.getUpgradeRequest().getParameterMap(); + List presenceTypes = queryParams.get("presenceType"); + + // We only support ONE presenceType + String presenceTypeName = presenceTypes == null || presenceTypes.isEmpty() ? null : presenceTypes.get(0); + + PresenceType presenceType = presenceTypeName == null ? null : PresenceType.fromString(presenceTypeName); + + // Make sure that if caller does give a presenceType, that it is a valid/known one. + if (presenceTypeName != null && presenceType == null) { + session.close(4003, "unknown presenceType: " + presenceTypeName); + return; + } + + // Save session's requested PresenceType, if given + if (presenceType != null) + sessionPresenceTypes.put(session, presenceType); + + List presenceInfo; + + synchronized (currentEntries) { + presenceInfo = currentEntries.entrySet().stream() + .filter(entry -> presenceType == null ? true : entry.getKey() == presenceType) + .flatMap(entry -> entry.getValue().entrySet().stream().map(innerEntry -> new PresenceInfo(entry.getKey(), innerEntry.getKey(), innerEntry.getValue()))) + .collect(Collectors.toList()); + } + + if (!sendPresenceInfo(session, presenceInfo)) { + session.close(4002, "websocket issue"); + return; + } + + super.onWebSocketConnect(session); + } + + @OnWebSocketClose + @Override + public void onWebSocketClose(Session session, int statusCode, String reason) { + // clean up + sessionPresenceTypes.remove(session); + + super.onWebSocketClose(session, statusCode, reason); + } + + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* ignored */ + } + + @OnWebSocketMessage + public void onWebSocketMessage(Session session, String message) { + /* ignored */ + } + + private boolean sendPresenceInfo(Session session, List presenceInfo) { + try { + StringWriter stringWriter = new StringWriter(); + marshall(stringWriter, presenceInfo); + + String output = stringWriter.toString(); + session.getRemote().sendStringByFuture(output); + } catch (IOException e) { + // No output this time? + return false; + } + + return true; + } + + private static void populateCurrentInfo(Repository repository) throws DataException { + // We want ALL PRESENCE transactions + + List presenceTransactionsData = repository.getTransactionRepository().getUnconfirmedTransactions(TransactionType.PRESENCE, null); + + for (TransactionData transactionData : presenceTransactionsData) { + PresenceTransactionData presenceData = (PresenceTransactionData) transactionData; + + PresenceType presenceType = presenceData.getPresenceType(); + + // Put/replace for this publickey making sure we keep newest timestamp + String pubKey58 = Base58.encode(presenceData.getCreatorPublicKey()); + long ourTimestamp = presenceData.getTimestamp(); + + mergePresence(presenceType, pubKey58, ourTimestamp); + } + } + + private static long mergePresence(PresenceType presenceType, String pubKey58, long ourTimestamp) { + Map typedPubkeyTimestamps = currentEntries.computeIfAbsent(presenceType, someType -> Collections.synchronizedMap(new HashMap<>())); + return typedPubkeyTimestamps.compute(pubKey58, (somePubKey58, currentTimestamp) -> (currentTimestamp == null || currentTimestamp < ourTimestamp) ? ourTimestamp : currentTimestamp); + } + + private static void removeOldEntries() { + long now = NTP.getTime(); + + currentEntries.entrySet().forEach(entry -> { + long expiryThreshold = now - entry.getKey().getLifetime(); + entry.getValue().entrySet().removeIf(pubkeyTimestamp -> pubkeyTimestamp.getValue() < expiryThreshold); + }); + } + +} diff --git a/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java b/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java new file mode 100644 index 000000000..55969c6be --- /dev/null +++ b/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java @@ -0,0 +1,157 @@ +package org.qortal.api.websocket; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.qortal.controller.tradebot.TradeBot; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.Listener; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.Base58; + +@WebSocket +@SuppressWarnings("serial") +public class TradeBotWebSocket extends ApiWebSocket implements Listener { + + /** Cache of trade-bot entry states, keyed by trade-bot entry's "trade private key" (base58) */ + private static final Map PREVIOUS_STATES = new HashMap<>(); + + private static final Map sessionBlockchain = Collections.synchronizedMap(new HashMap<>()); + + @Override + public void configure(WebSocketServletFactory factory) { + factory.register(TradeBotWebSocket.class); + + try (final Repository repository = RepositoryManager.getRepository()) { + List tradeBotEntries = repository.getCrossChainRepository().getAllTradeBotData(); + if (tradeBotEntries == null) + // How do we properly fail here? + return; + + PREVIOUS_STATES.putAll(tradeBotEntries.stream().collect(Collectors.toMap(entry -> Base58.encode(entry.getTradePrivateKey()), TradeBotData::getStateValue))); + } catch (DataException e) { + // No output this time + } + + EventBus.INSTANCE.addListener(this::listen); + } + + @Override + public void listen(Event event) { + if (!(event instanceof TradeBot.StateChangeEvent)) + return; + + TradeBotData tradeBotData = ((TradeBot.StateChangeEvent) event).getTradeBotData(); + String tradePrivateKey58 = Base58.encode(tradeBotData.getTradePrivateKey()); + + synchronized (PREVIOUS_STATES) { + Integer previousStateValue = PREVIOUS_STATES.get(tradePrivateKey58); + if (previousStateValue != null && previousStateValue == tradeBotData.getStateValue()) + // Not changed + return; + + PREVIOUS_STATES.put(tradePrivateKey58, tradeBotData.getStateValue()); + } + + List tradeBotEntries = Collections.singletonList(tradeBotData); + + for (Session session : getSessions()) { + // Only send if this session has this/no preferred blockchain + String preferredBlockchain = sessionBlockchain.get(session); + + if (preferredBlockchain == null || preferredBlockchain.equals(tradeBotData.getForeignBlockchain())) + sendEntries(session, tradeBotEntries); + } + } + + @OnWebSocketConnect + @Override + public void onWebSocketConnect(Session session) { + Map> queryParams = session.getUpgradeRequest().getParameterMap(); + + List foreignBlockchains = queryParams.get("foreignBlockchain"); + final String foreignBlockchain = foreignBlockchains == null ? null : foreignBlockchains.get(0); + + // Make sure blockchain (if any) is valid + if (foreignBlockchain != null && SupportedBlockchain.fromString(foreignBlockchain) == null) { + session.close(4003, "unknown blockchain: " + foreignBlockchain); + return; + } + + // save session's preferred blockchain (if any) + sessionBlockchain.put(session, foreignBlockchain); + + // Send all known trade-bot entries + try (final Repository repository = RepositoryManager.getRepository()) { + List tradeBotEntries = repository.getCrossChainRepository().getAllTradeBotData(); + + // Optional filtering + if (foreignBlockchain != null) + tradeBotEntries = tradeBotEntries.stream() + .filter(tradeBotData -> tradeBotData.getForeignBlockchain().equals(foreignBlockchain)) + .collect(Collectors.toList()); + + if (!sendEntries(session, tradeBotEntries)) { + session.close(4002, "websocket issue"); + return; + } + } catch (DataException e) { + session.close(4001, "repository issue fetching trade-bot entries"); + return; + } + + super.onWebSocketConnect(session); + } + + @OnWebSocketClose + @Override + public void onWebSocketClose(Session session, int statusCode, String reason) { + // clean up + sessionBlockchain.remove(session); + + super.onWebSocketClose(session, statusCode, reason); + } + + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* ignored */ + } + + @OnWebSocketMessage + public void onWebSocketMessage(Session session, String message) { + /* ignored */ + } + + private boolean sendEntries(Session session, List tradeBotEntries) { + try { + StringWriter stringWriter = new StringWriter(); + marshall(stringWriter, tradeBotEntries); + + String output = stringWriter.toString(); + session.getRemote().sendStringByFuture(output); + } catch (IOException e) { + // No output this time? + return false; + } + + return true; + } + +} diff --git a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java new file mode 100644 index 000000000..186f79e35 --- /dev/null +++ b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java @@ -0,0 +1,351 @@ +package org.qortal.api.websocket; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.qortal.api.model.CrossChainOfferSummary; +import org.qortal.controller.Controller; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.AcctMode; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.Listener; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.ByteArray; +import org.qortal.utils.NTP; + +@WebSocket +@SuppressWarnings("serial") +public class TradeOffersWebSocket extends ApiWebSocket implements Listener { + + private static final Logger LOGGER = LogManager.getLogger(TradeOffersWebSocket.class); + + private static class CachedOfferInfo { + public final Map previousAtModes = new HashMap<>(); + + // OFFERING + public final Map currentSummaries = new HashMap<>(); + // REDEEMED/REFUNDED/CANCELLED + public final Map historicSummaries = new HashMap<>(); + } + // Manual synchronization + private static final Map cachedInfoByBlockchain = new HashMap<>(); + + private static final Predicate isHistoric = offerSummary + -> offerSummary.getMode() == AcctMode.REDEEMED + || offerSummary.getMode() == AcctMode.REFUNDED + || offerSummary.getMode() == AcctMode.CANCELLED; + + private static final Map sessionBlockchain = Collections.synchronizedMap(new HashMap<>()); + + @Override + public void configure(WebSocketServletFactory factory) { + factory.register(TradeOffersWebSocket.class); + + try (final Repository repository = RepositoryManager.getRepository()) { + populateCurrentSummaries(repository); + + populateHistoricSummaries(repository); + } catch (DataException e) { + // How to fail properly? + return; + } + + EventBus.INSTANCE.addListener(this::listen); + } + + @Override + public void listen(Event event) { + if (!(event instanceof Controller.NewBlockEvent)) + return; + + BlockData blockData = ((Controller.NewBlockEvent) event).getBlockData(); + + // Process any new info + + try (final Repository repository = RepositoryManager.getRepository()) { + // Find any new/changed trade ATs since this block + final Boolean isFinished = null; + final Integer dataByteOffset = null; + final Long expectedValue = null; + final Integer minimumFinalHeight = blockData.getHeight(); + + for (SupportedBlockchain blockchain : SupportedBlockchain.values()) { + Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(blockchain); + + List crossChainOfferSummaries = new ArrayList<>(); + + synchronized (cachedInfoByBlockchain) { + CachedOfferInfo cachedInfo = cachedInfoByBlockchain.computeIfAbsent(blockchain.name(), k -> new CachedOfferInfo()); + + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); + + List atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, + isFinished, dataByteOffset, expectedValue, minimumFinalHeight, + null, null, null); + + crossChainOfferSummaries.addAll(produceSummaries(repository, acct, atStates, blockData.getTimestamp())); + } + + // Remove any entries unchanged from last time + crossChainOfferSummaries.removeIf(offerSummary -> cachedInfo.previousAtModes.get(offerSummary.getQortalAtAddress()) == offerSummary.getMode()); + + // Skip to next blockchain if nothing has changed (for this blockchain) + if (crossChainOfferSummaries.isEmpty()) + continue; + + // Update + for (CrossChainOfferSummary offerSummary : crossChainOfferSummaries) { + String offerAtAddress = offerSummary.getQortalAtAddress(); + + cachedInfo.previousAtModes.put(offerAtAddress, offerSummary.getMode()); + LOGGER.trace(() -> String.format("Block height: %d, AT: %s, mode: %s", blockData.getHeight(), offerAtAddress, offerSummary.getMode().name())); + + switch (offerSummary.getMode()) { + case OFFERING: + cachedInfo.currentSummaries.put(offerAtAddress, offerSummary); + cachedInfo.historicSummaries.remove(offerAtAddress); + break; + + case REDEEMED: + case REFUNDED: + case CANCELLED: + cachedInfo.currentSummaries.remove(offerAtAddress); + cachedInfo.historicSummaries.put(offerAtAddress, offerSummary); + break; + + case TRADING: + cachedInfo.currentSummaries.remove(offerAtAddress); + cachedInfo.historicSummaries.remove(offerAtAddress); + break; + } + } + + // Remove any historic offers that are over 24 hours old + final long tooOldTimestamp = NTP.getTime() - 24 * 60 * 60 * 1000L; + cachedInfo.historicSummaries.values().removeIf(historicSummary -> historicSummary.getTimestamp() < tooOldTimestamp); + } + + // Notify sessions + for (Session session : getSessions()) { + // Only send if this session has this/no preferred blockchain + String preferredBlockchain = sessionBlockchain.get(session); + + if (preferredBlockchain == null || preferredBlockchain.equals(blockchain.name())) + sendOfferSummaries(session, crossChainOfferSummaries); + } + + } + } catch (DataException e) { + // No output this time + } + } + + @OnWebSocketConnect + @Override + public void onWebSocketConnect(Session session) { + Map> queryParams = session.getUpgradeRequest().getParameterMap(); + final boolean includeHistoric = queryParams.get("includeHistoric") != null; + + List foreignBlockchains = queryParams.get("foreignBlockchain"); + final String foreignBlockchain = foreignBlockchains == null ? null : foreignBlockchains.get(0); + + // Make sure blockchain (if any) is valid + if (foreignBlockchain != null && SupportedBlockchain.fromString(foreignBlockchain) == null) { + session.close(4003, "unknown blockchain: " + foreignBlockchain); + return; + } + + // Save session's preferred blockchain, if given + if (foreignBlockchain != null) + sessionBlockchain.put(session, foreignBlockchain); + + List crossChainOfferSummaries = new ArrayList<>(); + + synchronized (cachedInfoByBlockchain) { + Collection cachedInfos; + + if (foreignBlockchain == null) + // No preferred blockchain, so iterate through all of them + cachedInfos = cachedInfoByBlockchain.values(); + else + cachedInfos = Collections.singleton(cachedInfoByBlockchain.computeIfAbsent(foreignBlockchain, k -> new CachedOfferInfo())); + + for (CachedOfferInfo cachedInfo : cachedInfos) { + crossChainOfferSummaries.addAll(cachedInfo.currentSummaries.values()); + + if (includeHistoric) + crossChainOfferSummaries.addAll(cachedInfo.historicSummaries.values()); + } + } + + if (!sendOfferSummaries(session, crossChainOfferSummaries)) { + session.close(4002, "websocket issue"); + return; + } + + super.onWebSocketConnect(session); + } + + @OnWebSocketClose + @Override + public void onWebSocketClose(Session session, int statusCode, String reason) { + // clean up + sessionBlockchain.remove(session); + + super.onWebSocketClose(session, statusCode, reason); + } + + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* ignored */ + } + + @OnWebSocketMessage + public void onWebSocketMessage(Session session, String message) { + /* ignored */ + } + + private boolean sendOfferSummaries(Session session, List crossChainOfferSummaries) { + try { + StringWriter stringWriter = new StringWriter(); + marshall(stringWriter, crossChainOfferSummaries); + + String output = stringWriter.toString(); + session.getRemote().sendStringByFuture(output); + } catch (IOException e) { + // No output this time? + return false; + } + + return true; + } + + private static void populateCurrentSummaries(Repository repository) throws DataException { + // We want ALL OFFERING trades + Boolean isFinished = Boolean.FALSE; + Long expectedValue = (long) AcctMode.OFFERING.value; + Integer minimumFinalHeight = null; + + for (SupportedBlockchain blockchain : SupportedBlockchain.values()) { + Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(blockchain); + + CachedOfferInfo cachedInfo = cachedInfoByBlockchain.computeIfAbsent(blockchain.name(), k -> new CachedOfferInfo()); + + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); + + Integer dataByteOffset = acct.getModeByteOffset(); + List initialAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash, + isFinished, dataByteOffset, expectedValue, minimumFinalHeight, + null, null, null); + + if (initialAtStates == null) + throw new DataException("Couldn't fetch current trades from repository"); + + // Save initial AT modes + cachedInfo.previousAtModes.putAll(initialAtStates.stream().collect(Collectors.toMap(ATStateData::getATAddress, atState -> AcctMode.OFFERING))); + + // Convert to offer summaries + cachedInfo.currentSummaries.putAll(produceSummaries(repository, acct, initialAtStates, null).stream() + .collect(Collectors.toMap(CrossChainOfferSummary::getQortalAtAddress, offerSummary -> offerSummary))); + } + } + } + + private static void populateHistoricSummaries(Repository repository) throws DataException { + // We want REDEEMED/REFUNDED/CANCELLED trades over the last 24 hours + long timestamp = System.currentTimeMillis() - 24 * 60 * 60 * 1000L; + int minimumFinalHeight = repository.getBlockRepository().getHeightFromTimestamp(timestamp); + + if (minimumFinalHeight == 0) + throw new DataException("Couldn't fetch block timestamp from repository"); + + Boolean isFinished = Boolean.TRUE; + Integer dataByteOffset = null; + Long expectedValue = null; + ++minimumFinalHeight; // because height is just *before* timestamp + + for (SupportedBlockchain blockchain : SupportedBlockchain.values()) { + Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(blockchain); + + CachedOfferInfo cachedInfo = cachedInfoByBlockchain.computeIfAbsent(blockchain.name(), k -> new CachedOfferInfo()); + + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); + + List historicAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash, + isFinished, dataByteOffset, expectedValue, minimumFinalHeight, + null, null, null); + + if (historicAtStates == null) + throw new DataException("Couldn't fetch historic trades from repository"); + + for (ATStateData historicAtState : historicAtStates) { + CrossChainOfferSummary historicOfferSummary = produceSummary(repository, acct, historicAtState, null); + + if (!isHistoric.test(historicOfferSummary)) + continue; + + // Add summary to initial burst + cachedInfo.historicSummaries.put(historicOfferSummary.getQortalAtAddress(), historicOfferSummary); + + // Save initial AT mode + cachedInfo.previousAtModes.put(historicOfferSummary.getQortalAtAddress(), historicOfferSummary.getMode()); + } + } + } + } + + private static CrossChainOfferSummary produceSummary(Repository repository, ACCT acct, ATStateData atState, Long timestamp) throws DataException { + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState); + + long atStateTimestamp; + + if (crossChainTradeData.mode == AcctMode.OFFERING) + // We want when trade was created, not when it was last updated + atStateTimestamp = crossChainTradeData.creationTimestamp; + else + atStateTimestamp = timestamp != null ? timestamp : repository.getBlockRepository().getTimestampFromHeight(atState.getHeight()); + + return new CrossChainOfferSummary(crossChainTradeData, atStateTimestamp); + } + + private static List produceSummaries(Repository repository, ACCT acct, List atStates, Long timestamp) throws DataException { + List offerSummaries = new ArrayList<>(); + + for (ATStateData atState : atStates) + offerSummaries.add(produceSummary(repository, acct, atState, timestamp)); + + return offerSummaries; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java new file mode 100644 index 000000000..4a02f0923 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -0,0 +1,101 @@ +package org.qortal.arbitrary; + +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.arbitrary.misc.Service; +import org.qortal.repository.DataException; +import org.qortal.utils.NTP; + +import java.io.IOException; + +public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource { + + private final Long creationTimestamp; + private Long buildStartTimestamp = null; + private Long buildEndTimestamp = null; + private Integer priority = 0; + private boolean failed = false; + + private static int HIGH_PRIORITY_THRESHOLD = 5; + + /* The maximum amount of time to spend on a single build */ + // TODO: interrupt an in-progress build + public static long BUILD_TIMEOUT = 60*1000L; // 60 seconds + /* The amount of time to remember that a build has failed, to avoid retries */ + public static long FAILURE_TIMEOUT = 5*60*1000L; // 5 minutes + + public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) { + super(resourceId, resourceIdType, service, identifier); + + this.creationTimestamp = NTP.getTime(); + } + + public void prepareForBuild() { + this.buildStartTimestamp = NTP.getTime(); + } + + public void build() throws IOException, DataException, MissingDataException { + Long now = NTP.getTime(); + if (now == null) { + this.buildStartTimestamp = null; + throw new DataException("NTP time hasn't synced yet"); + } + + if (this.buildStartTimestamp == null) { + this.buildStartTimestamp = now; + } + ArbitraryDataReader arbitraryDataReader = + new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier); + + try { + arbitraryDataReader.loadSynchronously(true); + } finally { + this.buildEndTimestamp = NTP.getTime(); + } + } + + public boolean isBuilding() { + return this.buildStartTimestamp != null; + } + + public boolean isQueued() { + return this.buildStartTimestamp == null; + } + + public boolean hasReachedBuildTimeout(Long now) { + if (now == null || this.creationTimestamp == null) { + return true; + } + return now - this.creationTimestamp > BUILD_TIMEOUT; + } + + public boolean hasReachedFailureTimeout(Long now) { + if (now == null || this.buildStartTimestamp == null) { + return true; + } + return now - this.buildStartTimestamp > FAILURE_TIMEOUT; + } + + public Long getBuildStartTimestamp() { + return this.buildStartTimestamp; + } + + public Integer getPriority() { + if (this.priority != null) { + return this.priority; + } + return 0; + } + + public void setPriority(Integer priority) { + this.priority = priority; + } + + public boolean isHighPriority() { + return this.priority >= HIGH_PRIORITY_THRESHOLD; + } + + public void setFailed(boolean failed) { + this.failed = failed; + } +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java new file mode 100644 index 000000000..4f0e38350 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -0,0 +1,280 @@ +package org.qortal.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache; +import org.qortal.arbitrary.misc.Service; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.Method; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; +import org.qortal.settings.Settings; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class ArbitraryDataBuilder { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuilder.class); + + private final String name; + private final Service service; + private final String identifier; + + private boolean canRequestMissingFiles; + + private List transactions; + private ArbitraryTransactionData latestPutTransaction; + private final List paths; + private byte[] latestSignature; + private Path finalPath; + private int layerCount; + + public ArbitraryDataBuilder(String name, Service service, String identifier) { + this.name = name; + this.service = service; + this.identifier = identifier; + this.paths = new ArrayList<>(); + + // By default we can request missing files + // Callers can use setCanRequestMissingFiles(false) to prevent it + this.canRequestMissingFiles = true; + } + + /** + * Process transactions, but do not build anything + * This is useful for checking the status of a given resource + * + * @throws DataException + * @throws IOException + * @throws MissingDataException + */ + public void process() throws DataException, IOException, MissingDataException { + this.fetchTransactions(); + this.validateTransactions(); + this.processTransactions(); + this.validatePaths(); + this.findLatestSignature(); + } + + /** + * Build the latest state of a given resource + * + * @throws DataException + * @throws IOException + * @throws MissingDataException + */ + public void build() throws DataException, IOException, MissingDataException { + this.process(); + this.buildLatestState(); + this.cacheLatestSignature(); + } + + private void fetchTransactions() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Get the most recent PUT + ArbitraryTransactionData latestPut = repository.getArbitraryRepository() + .getLatestTransaction(this.name, this.service, Method.PUT, this.identifier); + if (latestPut == null) { + String message = String.format("Couldn't find PUT transaction for name %s, service %s and identifier %s", + this.name, this.service, this.identifierString()); + throw new DataException(message); + } + this.latestPutTransaction = latestPut; + + // Load all transactions since the latest PUT + List transactionDataList = repository.getArbitraryRepository() + .getArbitraryTransactions(this.name, this.service, this.identifier, latestPut.getTimestamp()); + + this.transactions = transactionDataList; + this.layerCount = transactionDataList.size(); + } + } + + private void validateTransactions() throws DataException { + List transactionDataList = new ArrayList<>(this.transactions); + ArbitraryTransactionData latestPut = this.latestPutTransaction; + + if (latestPut == null) { + throw new DataException("Cannot PATCH without existing PUT. Deploy using PUT first."); + } + if (latestPut.getMethod() != Method.PUT) { + throw new DataException("Expected PUT but received PATCH"); + } + if (transactionDataList.size() == 0) { + throw new DataException(String.format("No transactions found for name %s, service %s, " + + "identifier: %s, since %d", name, service, this.identifierString(), latestPut.getTimestamp())); + } + + // Verify that the signature of the first transaction matches the latest PUT + ArbitraryTransactionData firstTransaction = transactionDataList.get(0); + if (!Arrays.equals(firstTransaction.getSignature(), latestPut.getSignature())) { + throw new DataException("First transaction did not match latest PUT transaction"); + } + + // Remove the first transaction, as it should be the only PUT + transactionDataList.remove(0); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + if (transactionData == null) { + throw new DataException("Transaction not found"); + } + if (transactionData.getMethod() != Method.PATCH) { + throw new DataException("Expected PATCH but received PUT"); + } + } + } + + private void processTransactions() throws IOException, DataException, MissingDataException { + List transactionDataList = new ArrayList<>(this.transactions); + + int count = 0; + for (ArbitraryTransactionData transactionData : transactionDataList) { + LOGGER.trace("Found arbitrary transaction {}", Base58.encode(transactionData.getSignature())); + count++; + + // Build the data file, overwriting anything that was previously there + String sig58 = Base58.encode(transactionData.getSignature()); + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, + this.service, this.identifier); + arbitraryDataReader.setTransactionData(transactionData); + arbitraryDataReader.setCanRequestMissingFiles(this.canRequestMissingFiles); + boolean hasMissingData = false; + try { + arbitraryDataReader.loadSynchronously(true); + } + catch (MissingDataException e) { + hasMissingData = true; + } + + // Handle missing data + if (hasMissingData) { + if (!this.canRequestMissingFiles) { + throw new MissingDataException("Files are missing but were not requested."); + } + if (count == transactionDataList.size()) { + // This is the final transaction in the list, so we need to fail + throw new MissingDataException("Requesting missing files. Please wait and try again."); + } + // There are more transactions, so we should process them to give them the opportunity to request data + continue; + } + + // By this point we should have all data needed to build the layers + Path path = arbitraryDataReader.getFilePath(); + if (path == null) { + throw new DataException(String.format("Null path when building data from transaction %s", sig58)); + } + if (!Files.exists(path)) { + throw new DataException(String.format("Path doesn't exist when building data from transaction %s", sig58)); + } + paths.add(path); + } + } + + private void findLatestSignature() throws DataException { + if (this.transactions.size() == 0) { + throw new DataException("Unable to find latest signature from empty transaction list"); + } + + // Find the latest signature + ArbitraryTransactionData latestTransaction = this.transactions.get(this.transactions.size() - 1); + if (latestTransaction == null) { + throw new DataException("Unable to find latest signature from null transaction"); + } + + this.latestSignature = latestTransaction.getSignature(); + } + + private void validatePaths() throws DataException { + if (this.paths.isEmpty()) { + throw new DataException("No paths available from which to build latest state"); + } + } + + private void buildLatestState() throws IOException, DataException { + if (this.paths.size() == 1) { + // No patching needed + this.finalPath = this.paths.get(0); + return; + } + + Path pathBefore = this.paths.get(0); + boolean validateAllLayers = Settings.getInstance().shouldValidateAllDataLayers(); + + // Loop from the second path onwards + for (int i=1; i addedPaths; + private final List modifiedPaths; + private final List removedPaths; + + private int totalFileCount; + private ArbitraryDataMetadataPatch metadata; + + public ArbitraryDataDiff(Path pathBefore, Path pathAfter, byte[] previousSignature) throws DataException { + this.pathBefore = pathBefore; + this.pathAfter = pathAfter; + this.previousSignature = previousSignature; + + this.addedPaths = new ArrayList<>(); + this.modifiedPaths = new ArrayList<>(); + this.removedPaths = new ArrayList<>(); + + this.createRandomIdentifier(); + this.createOutputDirectory(); + } + + public void compute() throws IOException, DataException { + try { + this.preExecute(); + this.hashPreviousState(); + this.findAddedOrModifiedFiles(); + this.findRemovedFiles(); + this.validate(); + this.hashCurrentState(); + this.writeMetadata(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + LOGGER.debug("Generating diff..."); + } + + private void postExecute() { + + } + + private void createRandomIdentifier() { + this.identifier = UUID.randomUUID().toString(); + } + + private void createOutputDirectory() throws DataException { + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + Path tempDir = Paths.get(baseDir, "diff", this.identifier); + try { + Files.createDirectories(tempDir); + } catch (IOException e) { + throw new DataException("Unable to create temp directory"); + } + this.diffPath = tempDir; + } + + private void hashPreviousState() throws IOException, DataException { + ArbitraryDataDigest digest = new ArbitraryDataDigest(this.pathBefore); + digest.compute(); + this.previousHash = digest.getHash(); + } + + private void findAddedOrModifiedFiles() throws IOException { + try { + final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); + final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); + final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); + final ArbitraryDataDiff diff = this; + + // Check for additions or modifications + Files.walkFileTree(this.pathAfter, new FileVisitor<>() { + + @Override + public FileVisitResult preVisitDirectory(Path after, BasicFileAttributes attrs) { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path afterPathAbsolute, BasicFileAttributes attrs) throws IOException { + Path afterPathRelative = pathAfterAbsolute.relativize(afterPathAbsolute.toAbsolutePath()); + Path beforePathAbsolute = pathBeforeAbsolute.resolve(afterPathRelative); + + if (afterPathRelative.startsWith(".qortal")) { + // Ignore the .qortal metadata folder + return FileVisitResult.CONTINUE; + } + + boolean wasAdded = false; + boolean wasModified = false; + + if (!Files.exists(beforePathAbsolute)) { + LOGGER.trace("File was added: {}", afterPathRelative.toString()); + diff.addedPaths.add(afterPathRelative); + wasAdded = true; + } + else if (Files.size(afterPathAbsolute) != Files.size(beforePathAbsolute)) { + // Check file size first because it's quicker + LOGGER.trace("File size was modified: {}", afterPathRelative.toString()); + wasModified = true; + } + else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(afterPathAbsolute), ArbitraryDataDiff.digestFromPath(beforePathAbsolute))) { + // Check hashes as a last resort + LOGGER.trace("File contents were modified: {}", afterPathRelative.toString()); + wasModified = true; + } + + if (wasAdded) { + diff.copyFilePathToBaseDir(afterPathAbsolute, diffPathAbsolute, afterPathRelative); + } + if (wasModified) { + try { + diff.pathModified(beforePathAbsolute, afterPathAbsolute, afterPathRelative, diffPathAbsolute); + } catch (DataException e) { + // We can only throw IOExceptions because we are overriding FileVisitor.visitFile() + throw new IOException(e); + } + } + + // Keep a tally of the total number of files to help with decision making + diff.totalFileCount++; + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e){ + LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage()); + // TODO: throw exception? + return FileVisitResult.TERMINATE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) { + return FileVisitResult.CONTINUE; + } + + }); + } catch (IOException e) { + LOGGER.info("IOException when walking through file tree: {}", e.getMessage()); + throw(e); + } + } + + private void findRemovedFiles() throws IOException { + try { + final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); + final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); + final ArbitraryDataDiff diff = this; + + // Check for removals + Files.walkFileTree(this.pathBefore, new FileVisitor<>() { + + @Override + public FileVisitResult preVisitDirectory(Path before, BasicFileAttributes attrs) { + Path directoryPathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath()); + Path directoryPathAfter = pathAfterAbsolute.resolve(directoryPathBefore); + + if (directoryPathBefore.startsWith(".qortal")) { + // Ignore the .qortal metadata folder + return FileVisitResult.CONTINUE; + } + + if (!Files.exists(directoryPathAfter)) { + LOGGER.trace("Directory was removed: {}", directoryPathAfter.toString()); + diff.removedPaths.add(directoryPathBefore); + // TODO: we might need to mark directories differently to files + } + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path before, BasicFileAttributes attrs) { + Path filePathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath()); + Path filePathAfter = pathAfterAbsolute.resolve(filePathBefore); + + if (filePathBefore.startsWith(".qortal")) { + // Ignore the .qortal metadata folder + return FileVisitResult.CONTINUE; + } + + if (!Files.exists(filePathAfter)) { + LOGGER.trace("File was removed: {}", filePathBefore.toString()); + diff.removedPaths.add(filePathBefore); + } + + // Keep a tally of the total number of files to help with decision making + diff.totalFileCount++; + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e){ + LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage()); + // TODO: throw exception? + return FileVisitResult.TERMINATE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) { + return FileVisitResult.CONTINUE; + } + + }); + } catch (IOException e) { + throw new IOException(String.format("IOException when walking through file tree: %s", e.getMessage())); + } + } + + private void validate() throws DataException { + if (this.addedPaths.isEmpty() && this.modifiedPaths.isEmpty() && this.removedPaths.isEmpty()) { + throw new DataException("Current state matches previous state. Nothing to do."); + } + } + + private void hashCurrentState() throws IOException, DataException { + ArbitraryDataDigest digest = new ArbitraryDataDigest(this.pathAfter); + digest.compute(); + this.currentHash = digest.getHash(); + } + + private void writeMetadata() throws IOException, DataException { + ArbitraryDataMetadataPatch metadata = new ArbitraryDataMetadataPatch(this.diffPath); + metadata.setAddedPaths(this.addedPaths); + metadata.setModifiedPaths(this.modifiedPaths); + metadata.setRemovedPaths(this.removedPaths); + metadata.setPreviousSignature(this.previousSignature); + metadata.setPreviousHash(this.previousHash); + metadata.setCurrentHash(this.currentHash); + metadata.write(); + this.metadata = metadata; + } + + + private void pathModified(Path beforePathAbsolute, Path afterPathAbsolute, Path afterPathRelative, + Path destinationBasePathAbsolute) throws IOException, DataException { + + Path destination = Paths.get(destinationBasePathAbsolute.toString(), afterPathRelative.toString()); + long beforeSize = Files.size(beforePathAbsolute); + long afterSize = Files.size(afterPathAbsolute); + DiffType diffType; + + if (beforeSize > MAX_DIFF_FILE_SIZE || afterSize > MAX_DIFF_FILE_SIZE) { + // Files are large, so don't attempt a diff + this.copyFilePathToBaseDir(afterPathAbsolute, destinationBasePathAbsolute, afterPathRelative); + diffType = DiffType.COMPLETE_FILE; + } + else { + // Attempt to create patch using java-diff-utils + UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(beforePathAbsolute, afterPathAbsolute, destination); + unifiedDiffPatch.create(); + if (unifiedDiffPatch.isValid()) { + diffType = DiffType.UNIFIED_DIFF; + } + else { + // Diff failed validation, so copy the whole file instead + this.copyFilePathToBaseDir(afterPathAbsolute, destinationBasePathAbsolute, afterPathRelative); + diffType = DiffType.COMPLETE_FILE; + } + } + + ModifiedPath modifiedPath = new ModifiedPath(afterPathRelative, diffType); + this.modifiedPaths.add(modifiedPath); + } + + private void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException { + if (!Files.exists(source)) { + throw new IOException(String.format("File not found: %s", source.toString())); + } + + // Ensure parent folders exist in the destination + Path dest = Paths.get(base.toString(), relativePath.toString()); + File file = new File(dest.toString()); + File parent = file.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + + LOGGER.trace("Copying {} to {}", source, dest); + Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + } + + + public Path getDiffPath() { + return this.diffPath; + } + + public int getTotalFileCount() { + return this.totalFileCount; + } + + public ArbitraryDataMetadataPatch getMetadata() { + return this.metadata; + } + + + // Utils + + private static byte[] digestFromPath(Path path) { + try { + return Crypto.digest(path.toFile()); + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java new file mode 100644 index 000000000..9703b231e --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java @@ -0,0 +1,73 @@ +package org.qortal.arbitrary; + +import org.qortal.repository.DataException; +import org.qortal.utils.Base58; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class ArbitraryDataDigest { + + private final Path path; + private byte[] hash; + + public ArbitraryDataDigest(Path path) { + this.path = path; + } + + public void compute() throws IOException, DataException { + List allPaths = Files.walk(path).filter(Files::isRegularFile).sorted().collect(Collectors.toList()); + Path basePathAbsolute = this.path.toAbsolutePath(); + + MessageDigest sha256; + try { + sha256 = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new DataException("SHA-256 hashing algorithm unavailable"); + } + + for (Path path : allPaths) { + // We need to work with paths relative to the base path, to ensure the same hash + // is generated on different systems + Path relativePath = basePathAbsolute.relativize(path.toAbsolutePath()); + + // Exclude Qortal folder since it can be different each time + // We only care about hashing the actual user data + if (relativePath.startsWith(".qortal/")) { + continue; + } + + // Hash path + byte[] filePathBytes = relativePath.toString().getBytes(StandardCharsets.UTF_8); + sha256.update(filePathBytes); + + // Hash contents + byte[] fileContent = Files.readAllBytes(path); + sha256.update(fileContent); + } + this.hash = sha256.digest(); + } + + public boolean isHashValid(byte[] hash) { + return Arrays.equals(hash, this.hash); + } + + public byte[] getHash() { + return this.hash; + } + + public String getHash58() { + if (this.hash == null) { + return null; + } + return Base58.encode(this.hash); + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java new file mode 100644 index 000000000..6b29de8da --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -0,0 +1,793 @@ +package org.qortal.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; +import org.qortal.crypto.Crypto; +import org.qortal.repository.DataException; +import org.qortal.settings.Settings; +import org.qortal.utils.Base58; +import org.qortal.utils.FilesystemUtils; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.*; +import java.util.stream.Stream; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + + +public class ArbitraryDataFile { + + // Validation results + public enum ValidationResult { + OK(1), + FILE_TOO_LARGE(10), + FILE_NOT_FOUND(11); + + public final int value; + + private static final Map map = stream(ArbitraryDataFile.ValidationResult.values()).collect(toMap(result -> result.value, result -> result)); + + ValidationResult(int value) { + this.value = value; + } + + public static ArbitraryDataFile.ValidationResult valueOf(int value) { + return map.get(value); + } + } + + // Resource ID types + public enum ResourceIdType { + SIGNATURE, + FILE_HASH, + TRANSACTION_DATA, + NAME + } + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class); + + public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB + public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB + public static int SHORT_DIGEST_LENGTH = 8; + + protected Path filePath; + protected String hash58; + protected byte[] signature; + private ArrayList chunks; + private byte[] secret; + + // Metadata + private byte[] metadataHash; + private ArbitraryDataFile metadataFile; + private ArbitraryDataTransactionMetadata metadata; + + + public ArbitraryDataFile() { + } + + public ArbitraryDataFile(String hash58, byte[] signature) throws DataException { + this.createDataDirectory(); + this.filePath = ArbitraryDataFile.getOutputFilePath(hash58, signature, false); + this.chunks = new ArrayList<>(); + this.hash58 = hash58; + this.signature = signature; + } + + public ArbitraryDataFile(byte[] fileContent, byte[] signature) throws DataException { + if (fileContent == null) { + LOGGER.error("fileContent is null"); + return; + } + + this.hash58 = Base58.encode(Crypto.digest(fileContent)); + this.signature = signature; + LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length)); + + Path outputFilePath = getOutputFilePath(this.hash58, signature, true); + File outputFile = outputFilePath.toFile(); + try (FileOutputStream outputStream = new FileOutputStream(outputFile)) { + outputStream.write(fileContent); + this.filePath = outputFilePath; + // Verify hash + if (!this.hash58.equals(this.digest58())) { + LOGGER.error("Hash {} does not match file digest {}", this.hash58, this.digest58()); + this.delete(); + throw new DataException("Data file digest validation failed"); + } + } catch (IOException e) { + throw new DataException("Unable to write data to file"); + } + } + + public static ArbitraryDataFile fromHash58(String hash58, byte[] signature) throws DataException { + return new ArbitraryDataFile(hash58, signature); + } + + public static ArbitraryDataFile fromHash(byte[] hash, byte[] signature) throws DataException { + return ArbitraryDataFile.fromHash58(Base58.encode(hash), signature); + } + + public static ArbitraryDataFile fromPath(Path path, byte[] signature) { + if (path == null) { + return null; + } + File file = path.toFile(); + if (file.exists()) { + try { + byte[] digest = Crypto.digest(file); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + + // Copy file to data directory if needed + if (Files.exists(path) && !arbitraryDataFile.isInBaseDirectory(path)) { + arbitraryDataFile.copyToDataDirectory(path, signature); + } + // Or, if it's already in the data directory, we may need to move it + else if (!path.equals(arbitraryDataFile.getFilePath())) { + // Wrong path, so relocate (but don't cleanup, as the source folder may still be needed by the caller) + Path dest = arbitraryDataFile.getFilePath(); + FilesystemUtils.moveFile(path, dest, false); + } + return arbitraryDataFile; + + } catch (IOException | DataException e) { + LOGGER.error("Couldn't compute digest for ArbitraryDataFile"); + } + } + return null; + } + + public static ArbitraryDataFile fromFile(File file, byte[] signature) { + return ArbitraryDataFile.fromPath(Paths.get(file.getPath()), signature); + } + + private boolean createDataDirectory() { + // Create the data directory if it doesn't exist + String dataPath = Settings.getInstance().getDataPath(); + Path dataDirectory = Paths.get(dataPath); + try { + Files.createDirectories(dataDirectory); + } catch (IOException e) { + LOGGER.error("Unable to create data directory"); + return false; + } + return true; + } + + private Path copyToDataDirectory(Path sourcePath, byte[] signature) throws DataException { + if (this.hash58 == null || this.filePath == null) { + return null; + } + Path outputFilePath = getOutputFilePath(this.hash58, signature, true); + sourcePath = sourcePath.toAbsolutePath(); + Path destPath = outputFilePath.toAbsolutePath(); + try { + return Files.copy(sourcePath, destPath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new DataException(String.format("Unable to copy file %s to data directory %s", sourcePath, destPath)); + } + } + + public static Path getOutputFilePath(String hash58, byte[] signature, boolean createDirectories) throws DataException { + Path directory; + + if (hash58 == null) { + return null; + } + if (signature != null) { + // Key by signature + String signature58 = Base58.encode(signature); + String sig58First2Chars = signature58.substring(0, 2).toLowerCase(); + String sig58Next2Chars = signature58.substring(2, 4).toLowerCase(); + directory = Paths.get(Settings.getInstance().getDataPath(), sig58First2Chars, sig58Next2Chars, signature58); + } + else { + // Put files without signatures in a "_misc" directory, and the files will be relocated later + String hash58First2Chars = hash58.substring(0, 2).toLowerCase(); + String hash58Next2Chars = hash58.substring(2, 4).toLowerCase(); + directory = Paths.get(Settings.getInstance().getDataPath(), "_misc", hash58First2Chars, hash58Next2Chars); + } + + if (createDirectories) { + try { + Files.createDirectories(directory); + } catch (IOException e) { + throw new DataException("Unable to create data subdirectory"); + } + } + return Paths.get(directory.toString(), hash58); + } + + public ValidationResult isValid() { + try { + // Ensure the file exists on disk + if (!Files.exists(this.filePath)) { + LOGGER.error("File doesn't exist at path {}", this.filePath); + return ValidationResult.FILE_NOT_FOUND; + } + + // Validate the file size + long fileSize = Files.size(this.filePath); + if (fileSize > MAX_FILE_SIZE) { + LOGGER.error(String.format("ArbitraryDataFile is too large: %d bytes (max size: %d bytes)", fileSize, MAX_FILE_SIZE)); + return ArbitraryDataFile.ValidationResult.FILE_TOO_LARGE; + } + + } catch (IOException e) { + return ValidationResult.FILE_NOT_FOUND; + } + + return ValidationResult.OK; + } + + public void validateFileSize(long expectedSize) throws DataException { + // Verify that we can determine the file's size + long fileSize = 0; + try { + fileSize = Files.size(this.getFilePath()); + } catch (IOException e) { + throw new DataException(String.format("Couldn't get file size for transaction %s", Base58.encode(signature))); + } + + // Ensure the file's size matches the size reported by the transaction + if (fileSize != expectedSize) { + throw new DataException(String.format("File size mismatch for transaction %s", Base58.encode(signature))); + } + } + + private void addChunk(ArbitraryDataFileChunk chunk) { + this.chunks.add(chunk); + } + + private void addChunkHashes(List chunkHashes) throws DataException { + if (chunkHashes == null || chunkHashes.isEmpty()) { + return; + } + for (byte[] chunkHash : chunkHashes) { + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature); + this.addChunk(chunk); + } + } + + public List getChunkHashes() { + List hashes = new ArrayList<>(); + if (this.chunks == null || this.chunks.isEmpty()) { + return hashes; + } + + for (ArbitraryDataFileChunk chunkData : this.chunks) { + hashes.add(chunkData.getHash()); + } + + return hashes; + } + + public int split(int chunkSize) throws DataException { + try { + + File file = this.getFile(); + byte[] buffer = new byte[chunkSize]; + this.chunks = new ArrayList<>(); + + if (file != null) { + try (FileInputStream fileInputStream = new FileInputStream(file); + BufferedInputStream bis = new BufferedInputStream(fileInputStream)) { + + int numberOfBytes; + while ((numberOfBytes = bis.read(buffer)) > 0) { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + out.write(buffer, 0, numberOfBytes); + out.flush(); + + ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray(), this.signature); + ValidationResult validationResult = chunk.isValid(); + if (validationResult == ValidationResult.OK) { + this.chunks.add(chunk); + } else { + throw new DataException(String.format("Chunk %s is invalid", chunk)); + } + } + } + } + } + } catch (Exception e) { + throw new DataException("Unable to split file into chunks"); + } + + return this.chunks.size(); + } + + public boolean join() { + // Ensure we have chunks + if (this.chunks != null && this.chunks.size() > 0) { + + // Create temporary path for joined file + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + Path tempDir = Paths.get(baseDir, "join"); + try { + Files.createDirectories(tempDir); + } catch (IOException e) { + return false; + } + + // Join the chunks + Path outputPath = Paths.get(tempDir.toString(), this.chunks.get(0).digest58()); + File outputFile = new File(outputPath.toString()); + try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outputFile))) { + for (ArbitraryDataFileChunk chunk : this.chunks) { + File sourceFile = chunk.filePath.toFile(); + BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); + byte[] buffer = new byte[2048]; + int inSize; + while ((inSize = in.read(buffer)) != -1) { + out.write(buffer, 0, inSize); + } + in.close(); + } + out.close(); + + // Copy temporary file to data directory + this.filePath = this.copyToDataDirectory(outputPath, this.signature); + if (FilesystemUtils.pathInsideDataOrTempPath(outputPath)) { + Files.delete(outputPath); + } + + return true; + } catch (FileNotFoundException e) { + return false; + } catch (IOException | DataException e) { + return false; + } + } + return false; + } + + public boolean delete() { + // Delete the complete file + // ... but only if it's inside the Qortal data or temp directory + if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { + if (Files.exists(this.filePath)) { + try { + Files.delete(this.filePath); + this.cleanupFilesystem(); + LOGGER.debug("Deleted file {}", this.filePath); + return true; + } catch (IOException e) { + LOGGER.warn("Couldn't delete file at path {}", this.filePath); + } + } + } + return false; + } + + public boolean delete(int attempts) { + // Keep trying to delete the data until it is deleted, or we reach 10 attempts + for (int i=0; i 0) { + Iterator iterator = this.chunks.iterator(); + while (iterator.hasNext()) { + ArbitraryDataFileChunk chunk = (ArbitraryDataFileChunk) iterator.next(); + success = chunk.delete(); + iterator.remove(); + } + } + return success; + } + + public boolean deleteMetadata() { + if (this.metadataFile != null && this.metadataFile.exists()) { + return this.metadataFile.delete(); + } + return false; + } + + public boolean deleteAll() { + // Delete the complete file + boolean fileDeleted = this.delete(); + + // Delete the metadata file + boolean metadataDeleted = this.deleteMetadata(); + + // Delete the individual chunks + boolean chunksDeleted = this.deleteAllChunks(); + + return fileDeleted || metadataDeleted || chunksDeleted; + } + + protected void cleanupFilesystem() throws IOException { + // It is essential that use a separate path reference in this method + // as we don't want to modify this.filePath + Path path = this.filePath; + + FilesystemUtils.safeDeleteEmptyParentDirectories(path); + } + + public byte[] getBytes() { + try { + return Files.readAllBytes(this.filePath); + } catch (IOException e) { + LOGGER.error("Unable to read bytes for file"); + return null; + } + } + + + /* Helper methods */ + + private boolean isInBaseDirectory(Path filePath) { + Path path = filePath.toAbsolutePath(); + String dataPath = Settings.getInstance().getDataPath(); + String basePath = Paths.get(dataPath).toAbsolutePath().toString(); + return path.startsWith(basePath); + } + + public boolean exists() { + File file = this.filePath.toFile(); + return file.exists(); + } + + public boolean chunkExists(byte[] hash) { + for (ArbitraryDataFileChunk chunk : this.chunks) { + if (Arrays.equals(hash, chunk.getHash())) { + return chunk.exists(); + } + } + if (Arrays.equals(hash, this.metadataHash)) { + if (this.metadataFile != null) { + return this.metadataFile.exists(); + } + } + if (Arrays.equals(this.getHash(), hash)) { + return this.exists(); + } + return false; + } + + public boolean allChunksExist() { + try { + if (this.metadataHash == null) { + // We don't have any metadata so can't check if we have the chunks + // Even if this transaction has no chunks, we don't have the file either (already checked above) + return false; + } + + if (this.metadataFile == null) { + this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature); + } + + // If the metadata file doesn't exist, we can't check if we have the chunks + if (!metadataFile.getFilePath().toFile().exists()) { + return false; + } + + if (this.metadata == null) { + this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath())); + } + + // Read the metadata + List chunks = metadata.getChunks(); + for (byte[] chunkHash : chunks) { + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature); + if (!chunk.exists()) { + return false; + } + } + + return true; + + } catch (DataException e) { + // Something went wrong, so assume we don't have all the chunks + return false; + } + } + + public boolean anyChunksExist() throws DataException { + try { + if (this.metadataHash == null) { + // We don't have any metadata so can't check if we have the chunks + // Even if this transaction has no chunks, we don't have the file either (already checked above) + return false; + } + + if (this.metadataFile == null) { + this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature); + } + + // If the metadata file doesn't exist, we can't check if we have any chunks + if (!metadataFile.getFilePath().toFile().exists()) { + return false; + } + + if (this.metadata == null) { + this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath())); + } + + // Read the metadata + List chunks = metadata.getChunks(); + for (byte[] chunkHash : chunks) { + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature); + if (chunk.exists()) { + return true; + } + } + + return false; + + } catch (DataException e) { + // Something went wrong, so assume we don't have all the chunks + return false; + } + } + + public boolean allFilesExist() { + if (this.exists()) { + return true; + } + + // Complete file doesn't exist, so check the chunks + if (this.allChunksExist()) { + return true; + } + + return false; + } + + /** + * Retrieve a list of file hashes for this transaction that we do not hold locally + * + * @return a List of chunk hashes, or null if we are unable to determine what is missing + */ + public List missingHashes() { + List missingHashes = new ArrayList<>(); + try { + if (this.metadataHash == null) { + // We don't have any metadata so can't check if we have the chunks + // Even if this transaction has no chunks, we don't have the file either (already checked above) + return null; + } + + if (this.metadataFile == null) { + this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature); + } + + // If the metadata file doesn't exist, we can't check if we have the chunks + if (!metadataFile.getFilePath().toFile().exists()) { + return null; + } + + if (this.metadata == null) { + this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath())); + } + + // Read the metadata + List chunks = metadata.getChunks(); + for (byte[] chunkHash : chunks) { + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature); + if (!chunk.exists()) { + missingHashes.add(chunkHash); + } + } + + return missingHashes; + + } catch (DataException e) { + // Something went wrong, so we can't make a sensible decision + return null; + } + } + + public boolean containsChunk(byte[] hash) { + for (ArbitraryDataFileChunk chunk : this.chunks) { + if (Arrays.equals(hash, chunk.getHash())) { + return true; + } + } + return false; + } + + public long size() { + try { + return Files.size(this.filePath); + } catch (IOException e) { + return 0; + } + } + + public int chunkCount() { + return this.chunks.size(); + } + + public List getChunks() { + return this.chunks; + } + + public byte[] chunkHashes() throws DataException { + if (this.chunks != null && this.chunks.size() > 0) { + // Return null if we only have one chunk, with the same hash as the parent + if (Arrays.equals(this.digest(), this.chunks.get(0).digest())) { + return null; + } + + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for (ArbitraryDataFileChunk chunk : this.chunks) { + byte[] chunkHash = chunk.digest(); + if (chunkHash.length != 32) { + LOGGER.info("Invalid chunk hash length: {}", chunkHash.length); + throw new DataException("Invalid chunk hash length"); + } + outputStream.write(chunk.digest()); + } + return outputStream.toByteArray(); + } catch (IOException e) { + return null; + } + } + return null; + } + + public List chunkHashList() { + List chunks = new ArrayList<>(); + + if (this.chunks != null && this.chunks.size() > 0) { + // Return null if we only have one chunk, with the same hash as the parent + if (Arrays.equals(this.digest(), this.chunks.get(0).digest())) { + return null; + } + + try { + for (ArbitraryDataFileChunk chunk : this.chunks) { + byte[] chunkHash = chunk.digest(); + if (chunkHash.length != 32) { + LOGGER.info("Invalid chunk hash length: {}", chunkHash.length); + throw new DataException("Invalid chunk hash length"); + } + chunks.add(chunkHash); + } + return chunks; + + } catch (DataException e) { + return null; + } + } + return null; + } + + private void loadMetadata() throws DataException { + try { + this.metadata.read(); + + } catch (DataException | IOException e) { + throw new DataException(e); + } + } + + private File getFile() { + File file = this.filePath.toFile(); + if (file.exists()) { + return file; + } + return null; + } + + public Path getFilePath() { + return this.filePath; + } + + public byte[] digest() { + File file = this.getFile(); + if (file != null && file.exists()) { + try { + return Crypto.digest(file); + + } catch (IOException e) { + LOGGER.error("Couldn't compute digest for ArbitraryDataFile"); + } + } + return null; + } + + public String digest58() { + if (this.digest() != null) { + return Base58.encode(this.digest()); + } + return null; + } + + public String shortHash58() { + if (this.hash58 == null) { + return null; + } + return this.hash58.substring(0, Math.min(this.hash58.length(), SHORT_DIGEST_LENGTH)); + } + + public String getHash58() { + return this.hash58; + } + + public byte[] getHash() { + return Base58.decode(this.hash58); + } + + public String printChunks() { + String outputString = ""; + if (this.chunkCount() > 0) { + for (ArbitraryDataFileChunk chunk : this.chunks) { + if (outputString.length() > 0) { + outputString = outputString.concat(","); + } + outputString = outputString.concat(chunk.digest58()); + } + } + return outputString; + } + + public void setSecret(byte[] secret) { + this.secret = secret; + } + + public byte[] getSecret() { + return this.secret; + } + + public byte[] getSignature() { + return this.signature; + } + + public void setMetadataFile(ArbitraryDataFile metadataFile) { + this.metadataFile = metadataFile; + } + + public ArbitraryDataFile getMetadataFile() { + return this.metadataFile; + } + + public void setMetadataHash(byte[] hash) throws DataException { + this.metadataHash = hash; + + if (hash == null) { + return; + } + this.metadataFile = ArbitraryDataFile.fromHash(hash, this.signature); + if (metadataFile.exists()) { + this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath())); + this.addChunkHashes(this.metadata.getChunks()); + } + } + + public byte[] getMetadataHash() { + return this.metadataHash; + } + + public void setMetadata(ArbitraryDataTransactionMetadata metadata) throws DataException { + this.metadata = metadata; + this.loadMetadata(); + } + + @Override + public String toString() { + return this.shortHash58(); + } +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java new file mode 100644 index 000000000..b113fbbae --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java @@ -0,0 +1,54 @@ +package org.qortal.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.repository.DataException; +import org.qortal.utils.Base58; + +import java.io.IOException; +import java.nio.file.Files; + + +public class ArbitraryDataFileChunk extends ArbitraryDataFile { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileChunk.class); + + public ArbitraryDataFileChunk(String hash58, byte[] signature) throws DataException { + super(hash58, signature); + } + + public ArbitraryDataFileChunk(byte[] fileContent, byte[] signature) throws DataException { + super(fileContent, signature); + } + + public static ArbitraryDataFileChunk fromHash58(String hash58, byte[] signature) throws DataException { + return new ArbitraryDataFileChunk(hash58, signature); + } + + public static ArbitraryDataFileChunk fromHash(byte[] hash, byte[] signature) throws DataException { + return ArbitraryDataFileChunk.fromHash58(Base58.encode(hash), signature); + } + + @Override + public ValidationResult isValid() { + // DataChunk validation applies here too + ValidationResult superclassValidationResult = super.isValid(); + if (superclassValidationResult != ValidationResult.OK) { + return superclassValidationResult; + } + + try { + // Validate the file size (chunks have stricter limits) + long fileSize = Files.size(this.filePath); + if (fileSize > CHUNK_SIZE) { + LOGGER.error(String.format("DataFileChunk is too large: %d bytes (max chunk size: %d bytes)", fileSize, CHUNK_SIZE)); + return ValidationResult.FILE_TOO_LARGE; + } + + } catch (IOException e) { + return ValidationResult.FILE_NOT_FOUND; + } + + return ValidationResult.OK; + } +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java new file mode 100644 index 000000000..eab5c8280 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java @@ -0,0 +1,176 @@ +package org.qortal.arbitrary; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.ArbitraryDataDiff.*; +import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; +import org.qortal.arbitrary.patch.UnifiedDiffPatch; +import org.qortal.repository.DataException; +import org.qortal.settings.Settings; +import org.qortal.utils.FilesystemUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.*; +import java.util.List; +import java.util.UUID; + +public class ArbitraryDataMerge { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMerge.class); + + private final Path pathBefore; + private final Path pathAfter; + private Path mergePath; + private String identifier; + private ArbitraryDataMetadataPatch metadata; + + public ArbitraryDataMerge(Path pathBefore, Path pathAfter) { + this.pathBefore = pathBefore; + this.pathAfter = pathAfter; + } + + public void compute() throws IOException, DataException { + try { + this.preExecute(); + this.copyPreviousStateToMergePath(); + this.loadMetadata(); + this.applyDifferences(); + this.copyMetadata(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() throws DataException { + this.createRandomIdentifier(); + this.createOutputDirectory(); + } + + private void postExecute() { + + } + + private void createRandomIdentifier() { + this.identifier = UUID.randomUUID().toString(); + } + + private void createOutputDirectory() throws DataException { + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + Path tempDir = Paths.get(baseDir, "merge", this.identifier); + try { + Files.createDirectories(tempDir); + } catch (IOException e) { + throw new DataException("Unable to create temp directory"); + } + this.mergePath = tempDir; + } + + private void copyPreviousStateToMergePath() throws IOException { + ArbitraryDataMerge.copyDirPathToBaseDir(this.pathBefore, this.mergePath, Paths.get("")); + } + + private void loadMetadata() throws IOException, DataException { + this.metadata = new ArbitraryDataMetadataPatch(this.pathAfter); + this.metadata.read(); + } + + private void applyDifferences() throws IOException, DataException { + + List addedPaths = this.metadata.getAddedPaths(); + for (Path path : addedPaths) { + LOGGER.trace("File was added: {}", path.toString()); + Path filePath = Paths.get(this.pathAfter.toString(), path.toString()); + ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, path); + } + + List modifiedPaths = this.metadata.getModifiedPaths(); + for (ModifiedPath modifiedPath : modifiedPaths) { + LOGGER.trace("File was modified: {}", modifiedPath.toString()); + this.applyPatch(modifiedPath); + } + + List removedPaths = this.metadata.getRemovedPaths(); + for (Path path : removedPaths) { + LOGGER.trace("File was removed: {}", path.toString()); + ArbitraryDataMerge.deletePathInBaseDir(this.mergePath, path); + } + } + + private void applyPatch(ModifiedPath modifiedPath) throws IOException, DataException { + if (modifiedPath.getDiffType() == DiffType.UNIFIED_DIFF) { + // Create destination file from patch + UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(pathBefore, pathAfter, mergePath); + unifiedDiffPatch.apply(modifiedPath.getPath()); + } + else if (modifiedPath.getDiffType() == DiffType.COMPLETE_FILE) { + // Copy complete file + Path filePath = Paths.get(this.pathAfter.toString(), modifiedPath.getPath().toString()); + ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, modifiedPath.getPath()); + } + else { + throw new DataException(String.format("Unrecognized patch diff type: %s", modifiedPath.getDiffType())); + } + } + + private void copyMetadata() throws IOException { + Path filePath = Paths.get(this.pathAfter.toString(), ".qortal"); + ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, Paths.get(".qortal")); + } + + + private static void copyPathToBaseDir(Path source, Path base, Path relativePath) throws IOException { + if (!Files.exists(source)) { + throw new IOException(String.format("File not found: %s", source.toString())); + } + + File sourceFile = source.toFile(); + Path dest = Paths.get(base.toString(), relativePath.toString()); + LOGGER.trace("Copying {} to {}", source, dest); + + if (sourceFile.isFile()) { + Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + } + else if (sourceFile.isDirectory()) { + FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString()); + } + else { + throw new IOException(String.format("Invalid file: %s", source.toString())); + } + } + + private static void copyDirPathToBaseDir(Path source, Path base, Path relativePath) throws IOException { + if (!Files.exists(source)) { + throw new IOException(String.format("File not found: %s", source.toString())); + } + + Path dest = Paths.get(base.toString(), relativePath.toString()); + LOGGER.trace("Copying {} to {}", source, dest); + FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString()); + } + + private static void deletePathInBaseDir(Path base, Path relativePath) throws IOException { + Path dest = Paths.get(base.toString(), relativePath.toString()); + File file = new File(dest.toString()); + if (file.exists() && file.isFile()) { + if (FilesystemUtils.pathInsideDataOrTempPath(dest)) { + LOGGER.trace("Deleting file {}", dest); + Files.delete(dest); + } + } + if (file.exists() && file.isDirectory()) { + if (FilesystemUtils.pathInsideDataOrTempPath(dest)) { + LOGGER.trace("Deleting directory {}", dest); + FileUtils.deleteDirectory(file); + } + } + } + + public Path getMergePath() { + return this.mergePath; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java new file mode 100644 index 000000000..568549d8b --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -0,0 +1,566 @@ +package org.qortal.arbitrary; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; +import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; +import org.qortal.crypto.AES; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.settings.Settings; +import org.qortal.transform.Transformer; +import org.qortal.utils.ArbitraryTransactionUtils; +import org.qortal.utils.Base58; +import org.qortal.utils.FilesystemUtils; +import org.qortal.utils.ZipUtils; + +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.io.File; +import java.io.IOException; +import java.io.InvalidObjectException; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; + +public class ArbitraryDataReader { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataReader.class); + + private final String resourceId; + private final ResourceIdType resourceIdType; + private final Service service; + private final String identifier; + private ArbitraryTransactionData transactionData; + private String secret58; + private Path filePath; + private boolean canRequestMissingFiles; + + // Intermediate paths + private final Path workingPath; + private final Path uncompressedPath; + + // Stats (available for synchronous builds only) + private int layerCount; + private byte[] latestSignature; + + public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) { + // Ensure names are always lowercase + if (resourceIdType == ResourceIdType.NAME) { + resourceId = resourceId.toLowerCase(); + } + + // If identifier is a blank string, or reserved keyword "default", treat it as null + if (identifier == null || identifier.equals("") || identifier.equals("default")) { + identifier = null; + } + + this.resourceId = resourceId; + this.resourceIdType = resourceIdType; + this.service = service; + this.identifier = identifier; + + this.workingPath = this.buildWorkingPath(); + this.uncompressedPath = Paths.get(this.workingPath.toString(), "data"); + + // By default we can request missing files + // Callers can use setCanRequestMissingFiles(false) to prevent it + this.canRequestMissingFiles = true; + } + + private Path buildWorkingPath() { + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + String identifier = this.identifier != null ? this.identifier : "default"; + return Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier); + } + + public boolean isCachedDataAvailable() { + // If this resource is in the build queue then we shouldn't attempt to serve + // cached data, as it may not be fully built + if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(this.createQueueItem())) { + return false; + } + + // Not in the build queue - so check the cache itself + ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false, + this.resourceId, this.resourceIdType, this.service, this.identifier); + if (cache.isCachedDataAvailable()) { + this.filePath = this.uncompressedPath; + return true; + } + return false; + } + + public boolean isBuilding() { + return ArbitraryDataBuildManager.getInstance().isInBuildQueue(this.createQueueItem()); + } + + private ArbitraryDataBuildQueueItem createQueueItem() { + return new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service, this.identifier); + } + + /** + * loadAsynchronously + * + * Attempts to load the resource asynchronously + * This adds the build task to a queue, and the result will be cached when complete + * To check the status of the build, periodically call isCachedDataAvailable() + * Once it returns true, you can then use getFilePath() to access the data itself. + * + * @param overwrite - set to true to force rebuild an existing cache + * @return true if added or already present in queue; false if not + */ + public boolean loadAsynchronously(boolean overwrite, int priority) { + ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite, + this.resourceId, this.resourceIdType, this.service, this.identifier); + if (cache.isCachedDataAvailable()) { + // Use cached data + this.filePath = this.uncompressedPath; + return true; + } + + ArbitraryDataBuildQueueItem item = this.createQueueItem(); + item.setPriority(priority); + return ArbitraryDataBuildManager.getInstance().addToBuildQueue(item); + } + + /** + * loadSynchronously + * + * Attempts to load the resource synchronously + * Warning: this can block for a long time when building or fetching complex data + * If no exception is thrown, you can then use getFilePath() to access the data immediately after returning + * + * @param overwrite - set to true to force rebuild an existing cache + * @throws IOException + * @throws DataException + * @throws MissingDataException + */ + public void loadSynchronously(boolean overwrite) throws DataException, IOException, MissingDataException { + try { + ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite, + this.resourceId, this.resourceIdType, this.service, this.identifier); + if (cache.isCachedDataAvailable()) { + // Use cached data + this.filePath = this.uncompressedPath; + return; + } + + this.preExecute(); + this.deleteExistingFiles(); + this.fetch(); + this.decrypt(); + this.uncompress(); + this.validate(); + + } catch (DataException e) { + this.deleteWorkingDirectory(); + throw new DataException(e.getMessage()); + + } finally { + this.postExecute(); + } + } + + private void preExecute() throws DataException { + ArbitraryDataBuildManager.getInstance().setBuildInProgress(true); + this.checkEnabled(); + this.createWorkingDirectory(); + this.createUncompressedDirectory(); + } + + private void postExecute() { + ArbitraryDataBuildManager.getInstance().setBuildInProgress(false); + } + + private void checkEnabled() throws DataException { + if (!Settings.getInstance().isQdnEnabled()) { + throw new DataException("QDN is disabled in settings"); + } + } + + private void createWorkingDirectory() throws DataException { + try { + Files.createDirectories(this.workingPath); + } catch (IOException e) { + throw new DataException("Unable to create temp directory"); + } + } + + /** + * Working directory should only be deleted on failure, since it is currently used to + * serve a cached version of the resource for subsequent requests. + * @throws IOException + */ + private void deleteWorkingDirectory() throws IOException { + FilesystemUtils.safeDeleteDirectory(this.workingPath, true); + } + + private void createUncompressedDirectory() throws DataException { + try { + // Create parent directory + Files.createDirectories(this.uncompressedPath.getParent()); + // Ensure child directory doesn't already exist + FileUtils.deleteDirectory(this.uncompressedPath.toFile()); + + } catch (IOException e) { + throw new DataException("Unable to create uncompressed directory"); + } + } + + private void deleteExistingFiles() { + final Path uncompressedPath = this.uncompressedPath; + if (FilesystemUtils.pathInsideDataOrTempPath(uncompressedPath)) { + if (Files.exists(uncompressedPath)) { + LOGGER.trace("Attempting to delete path {}", this.uncompressedPath); + try { + Files.walkFileTree(uncompressedPath, new SimpleFileVisitor<>() { + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { + // Don't delete the parent directory, as we want to leave an empty folder + if (dir.compareTo(uncompressedPath) == 0) { + return FileVisitResult.CONTINUE; + } + + if (e == null) { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } else { + throw e; + } + } + + }); + } catch (IOException e) { + LOGGER.debug("Unable to delete file or directory: {}", e.getMessage()); + } + } + } + } + + private void fetch() throws DataException, IOException, MissingDataException { + switch (resourceIdType) { + + case FILE_HASH: + this.fetchFromFileHash(); + break; + + case NAME: + this.fetchFromName(); + break; + + case SIGNATURE: + this.fetchFromSignature(); + break; + + case TRANSACTION_DATA: + this.fetchFromTransactionData(this.transactionData); + break; + + default: + throw new DataException(String.format("Unknown resource ID type specified: %s", resourceIdType.toString())); + } + } + + private void fetchFromFileHash() throws DataException { + // Load data file directly from the hash (without a signature) + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId, null); + // Set filePath to the location of the ArbitraryDataFile + this.filePath = arbitraryDataFile.getFilePath(); + } + + private void fetchFromName() throws DataException, IOException, MissingDataException { + try { + + // Build the existing state using past transactions + ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.resourceId, this.service, this.identifier); + builder.build(); + Path builtPath = builder.getFinalPath(); + if (builtPath == null) { + throw new DataException("Unable to build path"); + } + + // Update stats + this.layerCount = builder.getLayerCount(); + this.latestSignature = builder.getLatestSignature(); + + // Set filePath to the builtPath + this.filePath = builtPath; + + } catch (InvalidObjectException e) { + // Hash validation failed. Invalidate the cache for this name, so it can be rebuilt + LOGGER.info("Deleting {}", this.workingPath.toString()); + FilesystemUtils.safeDeleteDirectory(this.workingPath, false); + throw(e); + } + } + + private void fetchFromSignature() throws DataException, IOException, MissingDataException { + + // Load the full transaction data from the database so we can access the file hashes + ArbitraryTransactionData transactionData; + try (final Repository repository = RepositoryManager.getRepository()) { + transactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(Base58.decode(resourceId)); + } + if (transactionData == null) { + throw new DataException(String.format("Transaction data not found for signature %s", this.resourceId)); + } + + this.fetchFromTransactionData(transactionData); + } + + private void fetchFromTransactionData(ArbitraryTransactionData transactionData) throws DataException, IOException, MissingDataException { + if (transactionData == null) { + throw new DataException(String.format("Transaction data not found for signature %s", this.resourceId)); + } + + // Load hashes + byte[] digest = transactionData.getData(); + byte[] metadataHash = transactionData.getMetadataHash(); + byte[] signature = transactionData.getSignature(); + + // Load secret + byte[] secret = transactionData.getSecret(); + if (secret != null) { + this.secret58 = Base58.encode(secret); + } + + // Load data file(s) + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData); + arbitraryDataFile.setMetadataHash(metadataHash); + + if (!arbitraryDataFile.allFilesExist()) { + if (ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName())) { + throw new DataException( + String.format("Unable to request missing data for file %s because the name is blocked", arbitraryDataFile)); + } + else { + // Ask the arbitrary data manager to fetch data for this transaction + String message; + if (this.canRequestMissingFiles) { + boolean requested = ArbitraryDataManager.getInstance().fetchData(transactionData); + + if (requested) { + message = String.format("Requested missing data for file %s", arbitraryDataFile); + } else { + message = String.format("Unable to reissue request for missing file %s for signature %s due to rate limit. Please try again later.", arbitraryDataFile, Base58.encode(transactionData.getSignature())); + } + } + else { + message = String.format("Missing data for file %s", arbitraryDataFile); + } + + // Throw a missing data exception, which allows subsequent layers to fetch data + LOGGER.trace(message); + throw new MissingDataException(message); + } + } + + if (arbitraryDataFile.allChunksExist() && !arbitraryDataFile.exists()) { + // We have all the chunks but not the complete file, so join them + arbitraryDataFile.join(); + } + + // If the complete file still doesn't exist then something went wrong + if (!arbitraryDataFile.exists()) { + throw new IOException(String.format("File doesn't exist: %s", arbitraryDataFile)); + } + // Ensure the complete hash matches the joined chunks + if (!Arrays.equals(arbitraryDataFile.digest(), digest)) { + // Delete the invalid file + arbitraryDataFile.delete(); + throw new DataException("Unable to validate complete file hash"); + } + // Ensure the file's size matches the size reported by the transaction (throws a DataException if not) + arbitraryDataFile.validateFileSize(transactionData.getSize()); + + // Set filePath to the location of the ArbitraryDataFile + this.filePath = arbitraryDataFile.getFilePath(); + } + + private void decrypt() throws DataException { + try { + // First try with explicit parameters (CBC mode with PKCS5 padding) + this.decryptUsingAlgo("AES/CBC/PKCS5Padding"); + + } catch (DataException e) { + // Something went wrong, so fall back to default AES params (necessary for legacy resource support) + this.decryptUsingAlgo("AES"); + + // TODO: delete files and block this resource if privateDataEnabled is false and the second attempt fails too + } + } + + private void decryptUsingAlgo(String algorithm) throws DataException { + // Decrypt if we have the secret key. + byte[] secret = this.secret58 != null ? Base58.decode(this.secret58) : null; + if (secret != null && secret.length == Transformer.AES256_LENGTH) { + try { + Path unencryptedPath = Paths.get(this.workingPath.toString(), "zipped.zip"); + SecretKey aesKey = new SecretKeySpec(secret, 0, secret.length, algorithm); + AES.decryptFile(algorithm, aesKey, this.filePath.toString(), unencryptedPath.toString()); + + // Replace filePath pointer with the encrypted file path + // Don't delete the original ArbitraryDataFile, as this is handled in the cleanup phase + this.filePath = unencryptedPath; + + } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException + | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) { + throw new DataException(String.format("Unable to decrypt file at path %s: %s", this.filePath, e.getMessage())); + } + } else { + // Assume it is unencrypted. This will be the case when we have built a custom path by combining + // multiple decrypted archives into a single state. + } + } + + private void uncompress() throws IOException, DataException { + if (this.filePath == null || !Files.exists(this.filePath)) { + throw new DataException("Can't uncompress non-existent file path"); + } + File file = new File(this.filePath.toString()); + if (file.isDirectory()) { + // Already a directory - nothing to uncompress + // We still need to copy the directory to its final destination if it's not already there + this.moveFilePathToFinalDestination(); + return; + } + + try { + // Default to ZIP compression - this is needed for previews + Compression compression = transactionData != null ? transactionData.getCompression() : Compression.ZIP; + + // Handle each type of compression + if (compression == Compression.ZIP) { + ZipUtils.unzip(this.filePath.toString(), this.uncompressedPath.getParent().toString()); + } + else if (compression == Compression.NONE) { + Files.createDirectories(this.uncompressedPath); + Path finalPath = Paths.get(this.uncompressedPath.toString(), "data"); + this.filePath.toFile().renameTo(finalPath.toFile()); + } + else { + throw new DataException(String.format("Unrecognized compression type: %s", transactionData.getCompression())); + } + } catch (IOException e) { + throw new DataException(String.format("Unable to unzip file: %s", e.getMessage())); + } + + if (!this.uncompressedPath.toFile().exists()) { + throw new DataException(String.format("Unable to unzip file: %s", this.filePath)); + } + + // Delete original compressed file + if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { + if (Files.exists(this.filePath)) { + Files.delete(this.filePath); + } + } + + // Replace filePath pointer with the uncompressed file path + this.filePath = this.uncompressedPath; + } + + private void validate() throws IOException, DataException { + if (this.service.isValidationRequired()) { + Service.ValidationResult result = this.service.validate(this.filePath); + if (result != Service.ValidationResult.OK) { + throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString())); + } + } + } + + + private void moveFilePathToFinalDestination() throws IOException, DataException { + if (this.filePath.compareTo(this.uncompressedPath) != 0) { + File source = new File(this.filePath.toString()); + File dest = new File(this.uncompressedPath.toString()); + if (!source.exists()) { + throw new DataException("Source directory doesn't exist"); + } + // Ensure destination directory doesn't exist + FileUtils.deleteDirectory(dest); + // Move files to destination + FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString()); + + try { + // Delete existing + if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { + File directory = new File(this.filePath.toString()); + FileUtils.deleteDirectory(directory); + } + + // ... and its parent directory if empty + Path parentDirectory = this.filePath.getParent(); + if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) { + Files.deleteIfExists(parentDirectory); + } + + } catch (DirectoryNotEmptyException e) { + // No need to log anything + } catch (IOException e) { + // This will eventually be cleaned up by a maintenance process, so log the error and continue + LOGGER.debug("Unable to cleanup directories: {}", e.getMessage()); + } + + // Finally, update filePath to point to uncompressedPath + this.filePath = this.uncompressedPath; + } + } + + + public void setTransactionData(ArbitraryTransactionData transactionData) { + this.transactionData = transactionData; + } + + public void setSecret58(String secret58) { + this.secret58 = secret58; + } + + public Path getFilePath() { + return this.filePath; + } + + public int getLayerCount() { + return this.layerCount; + } + + public byte[] getLatestSignature() { + return this.latestSignature; + } + + /** + * Use the below setter to ensure that we only read existing + * data without requesting any missing files, + * + * @param canRequestMissingFiles - whether or not fetching missing files is allowed + */ + public void setCanRequestMissingFiles(boolean canRequestMissingFiles) { + this.canRequestMissingFiles = canRequestMissingFiles; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java new file mode 100644 index 000000000..3bd47b266 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java @@ -0,0 +1,213 @@ +package org.qortal.arbitrary; + +import com.google.common.io.Resources; +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.api.HTMLParser; +import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.Controller; +import org.qortal.settings.Settings; + +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +public class ArbitraryDataRenderer { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataRenderer.class); + + private final String resourceId; + private final ResourceIdType resourceIdType; + private final Service service; + private String inPath; + private final String secret58; + private final String prefix; + private final boolean usePrefix; + private final boolean async; + private final HttpServletRequest request; + private final HttpServletResponse response; + private final ServletContext context; + + public ArbitraryDataRenderer(String resourceId, ResourceIdType resourceIdType, Service service, String inPath, + String secret58, String prefix, boolean usePrefix, boolean async, + HttpServletRequest request, HttpServletResponse response, ServletContext context) { + + this.resourceId = resourceId; + this.resourceIdType = resourceIdType; + this.service = service; + this.inPath = inPath; + this.secret58 = secret58; + this.prefix = prefix; + this.usePrefix = usePrefix; + this.async = async; + this.request = request; + this.response = response; + this.context = context; + } + + public HttpServletResponse render() { + if (!inPath.startsWith(File.separator)) { + inPath = File.separator + inPath; + } + + // Don't render data if QDN is disabled + if (!Settings.getInstance().isQdnEnabled()) { + return ArbitraryDataRenderer.getResponse(response, 500, "QDN is disabled in settings"); + } + + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, null); + arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only + try { + if (!arbitraryDataReader.isCachedDataAvailable()) { + // If async is requested, show a loading screen whilst build is in progress + if (async) { + arbitraryDataReader.loadAsynchronously(false, 10); + return this.getLoadingResponse(service, resourceId); + } + + // Otherwise, loop until we have data + int attempts = 0; + while (!Controller.isStopping()) { + attempts++; + if (!arbitraryDataReader.isBuilding()) { + try { + arbitraryDataReader.loadSynchronously(false); + break; + } catch (MissingDataException e) { + if (attempts > 5) { + // Give up after 5 attempts + return ArbitraryDataRenderer.getResponse(response, 404, "Data unavailable. Please try again later."); + } + } + } + Thread.sleep(3000L); + } + } + + } catch (Exception e) { + LOGGER.info(String.format("Unable to load %s %s: %s", service, resourceId, e.getMessage())); + return ArbitraryDataRenderer.getResponse(response, 500, "Error 500: Internal Server Error"); + } + + java.nio.file.Path path = arbitraryDataReader.getFilePath(); + if (path == null) { + return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found"); + } + String unzippedPath = path.toString(); + + try { + String filename = this.getFilename(unzippedPath, inPath); + String filePath = Paths.get(unzippedPath, filename).toString(); + + if (HTMLParser.isHtmlFile(filename)) { + // HTML file - needs to be parsed + byte[] data = Files.readAllBytes(Paths.get(filePath)); // TODO: limit file size that can be read into memory + HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data); + htmlParser.addAdditionalHeaderTags(); + response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline'; media-src 'self' blob:"); + response.setContentType(context.getMimeType(filename)); + response.setContentLength(htmlParser.getData().length); + response.getOutputStream().write(htmlParser.getData()); + } + else { + // Regular file - can be streamed directly + File file = new File(filePath); + FileInputStream inputStream = new FileInputStream(file); + response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline'; media-src 'self' blob:"); + response.setContentType(context.getMimeType(filename)); + int bytesRead, length = 0; + byte[] buffer = new byte[10240]; + while ((bytesRead = inputStream.read(buffer)) != -1) { + response.getOutputStream().write(buffer, 0, bytesRead); + length += bytesRead; + } + response.setContentLength(length); + inputStream.close(); + } + return response; + } catch (FileNotFoundException | NoSuchFileException e) { + LOGGER.info("Unable to serve file: {}", e.getMessage()); + if (inPath.equals("/")) { + // Delete the unzipped folder if no index file was found + try { + FileUtils.deleteDirectory(new File(unzippedPath)); + } catch (IOException ioException) { + LOGGER.debug("Unable to delete directory: {}", unzippedPath, e); + } + } + } catch (IOException e) { + LOGGER.info("Unable to serve file at path {}: {}", inPath, e.getMessage()); + } + + return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found"); + } + + private String getFilename(String directory, String userPath) { + if (userPath == null || userPath.endsWith("/") || userPath.equals("")) { + // Locate index file + List indexFiles = ArbitraryDataRenderer.indexFiles(); + for (String indexFile : indexFiles) { + Path path = Paths.get(directory, indexFile); + if (Files.exists(path)) { + return userPath + indexFile; + } + } + } + return userPath; + } + + private HttpServletResponse getLoadingResponse(Service service, String name) { + String responseString = ""; + URL url = Resources.getResource("loading/index.html"); + try { + responseString = Resources.toString(url, StandardCharsets.UTF_8); + + // Replace vars + responseString = responseString.replace("%%SERVICE%%", service.toString()); + responseString = responseString.replace("%%NAME%%", name); + + } catch (IOException e) { + LOGGER.info("Unable to show loading screen: {}", e.getMessage()); + } + return ArbitraryDataRenderer.getResponse(response, 503, responseString); + } + + public static HttpServletResponse getResponse(HttpServletResponse response, int responseCode, String responseString) { + try { + byte[] responseData = responseString.getBytes(); + response.setStatus(responseCode); + response.setContentLength(responseData.length); + response.getOutputStream().write(responseData); + } catch (IOException e) { + LOGGER.info("Error writing {} response", responseCode); + } + return response; + } + + public static List indexFiles() { + List indexFiles = new ArrayList<>(); + indexFiles.add("index.html"); + indexFiles.add("index.htm"); + indexFiles.add("default.html"); + indexFiles.add("default.htm"); + indexFiles.add("home.html"); + indexFiles.add("home.htm"); + return indexFiles; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java new file mode 100644 index 000000000..36bd8f4cd --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -0,0 +1,352 @@ +package org.qortal.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; +import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; +import org.qortal.data.arbitrary.ArbitraryResourceStatus; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.list.ResourceListManager; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.utils.ArbitraryTransactionUtils; +import org.qortal.utils.FilesystemUtils; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import static org.qortal.data.arbitrary.ArbitraryResourceStatus.Status; + +public class ArbitraryDataResource { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataResource.class); + + protected final String resourceId; + protected final ResourceIdType resourceIdType; + protected final Service service; + protected final String identifier; + + private List transactions; + private ArbitraryTransactionData latestPutTransaction; + private int layerCount; + private Integer localChunkCount = null; + private Integer totalChunkCount = null; + + public ArbitraryDataResource(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) { + this.resourceId = resourceId.toLowerCase(); + this.resourceIdType = resourceIdType; + this.service = service; + + // If identifier is a blank string, or reserved keyword "default", treat it as null + if (identifier == null || identifier.equals("") || identifier.equals("default")) { + identifier = null; + } + this.identifier = identifier; + } + + public ArbitraryResourceStatus getStatus(boolean quick) { + // Calculate the chunk counts + // Avoid this for "quick" statuses, to speed things up + if (!quick) { + this.calculateChunkCounts(); + } + + if (resourceIdType != ResourceIdType.NAME) { + // We only support statuses for resources with a name + return new ArbitraryResourceStatus(Status.UNSUPPORTED, this.localChunkCount, this.totalChunkCount); + } + + // Check if the name is blocked + if (ResourceListManager.getInstance() + .listContains("blockedNames", this.resourceId, false)) { + return new ArbitraryResourceStatus(Status.BLOCKED, this.localChunkCount, this.totalChunkCount); + } + + // Check if a build has failed + ArbitraryDataBuildQueueItem queueItem = + new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier); + if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) { + return new ArbitraryResourceStatus(Status.BUILD_FAILED, this.localChunkCount, this.totalChunkCount); + } + + // Firstly check the cache to see if it's already built + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader( + resourceId, resourceIdType, service, identifier); + if (arbitraryDataReader.isCachedDataAvailable()) { + return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount); + } + + // Check if we have all data locally for this resource + if (!this.allFilesDownloaded()) { + if (this.isDownloading()) { + return new ArbitraryResourceStatus(Status.DOWNLOADING, this.localChunkCount, this.totalChunkCount); + } + else if (this.isDataPotentiallyAvailable()) { + return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount); + } + return new ArbitraryResourceStatus(Status.MISSING_DATA, this.localChunkCount, this.totalChunkCount); + } + + // Check if there's a build in progress + if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) { + return new ArbitraryResourceStatus(Status.BUILDING, this.localChunkCount, this.totalChunkCount); + } + + // We have all data locally + return new ArbitraryResourceStatus(Status.DOWNLOADED, this.localChunkCount, this.totalChunkCount); + } + + public boolean delete() { + try { + this.fetchTransactions(); + + List transactionDataList = new ArrayList<>(this.transactions); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + byte[] hash = transactionData.getData(); + byte[] metadataHash = transactionData.getMetadataHash(); + byte[] signature = transactionData.getSignature(); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + + // Delete any chunks or complete files from each transaction + arbitraryDataFile.deleteAll(); + } + + // Also delete cached data for the entire resource + this.deleteCache(); + + // Invalidate the hosted transactions cache as we have removed an item + ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache(); + + return true; + + } catch (DataException | IOException e) { + return false; + } + } + + public void deleteCache() throws IOException { + // Don't delete anything if there's a build in progress + ArbitraryDataBuildQueueItem queueItem = + new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier); + if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) { + return; + } + + String baseDir = Settings.getInstance().getTempDataPath(); + String identifier = this.identifier != null ? this.identifier : "default"; + Path cachePath = Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier); + if (cachePath.toFile().exists()) { + boolean success = FilesystemUtils.safeDeleteDirectory(cachePath, true); + if (success) { + LOGGER.info("Cleared cache for resource {}", this.toString()); + } + } + } + + private boolean allFilesDownloaded() { + // Use chunk counts to speed things up if we can + if (this.localChunkCount != null && this.totalChunkCount != null && + this.localChunkCount >= this.totalChunkCount) { + return true; + } + + try { + this.fetchTransactions(); + + List transactionDataList = new ArrayList<>(this.transactions); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + if (!ArbitraryTransactionUtils.completeFileExists(transactionData) || + !ArbitraryTransactionUtils.allChunksExist(transactionData)) { + return false; + } + } + return true; + + } catch (DataException e) { + return false; + } + } + + private void calculateChunkCounts() { + try { + this.fetchTransactions(); + + List transactionDataList = new ArrayList<>(this.transactions); + int localChunkCount = 0; + int totalChunkCount = 0; + + for (ArbitraryTransactionData transactionData : transactionDataList) { + localChunkCount += ArbitraryTransactionUtils.ourChunkCount(transactionData); + totalChunkCount += ArbitraryTransactionUtils.totalChunkCount(transactionData); + } + + this.localChunkCount = localChunkCount; + this.totalChunkCount = totalChunkCount; + + } catch (DataException e) {} + } + + private boolean isRateLimited() { + try { + this.fetchTransactions(); + + List transactionDataList = new ArrayList<>(this.transactions); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + if (ArbitraryDataManager.getInstance().isSignatureRateLimited(transactionData.getSignature())) { + return true; + } + } + return true; + + } catch (DataException e) { + return false; + } + } + + /** + * Best guess as to whether data might be available + * This is only used to give an indication to the user of progress + * @return - whether data might be available on the network + */ + private boolean isDataPotentiallyAvailable() { + try { + this.fetchTransactions(); + Long now = NTP.getTime(); + if (now == null) { + return false; + } + + List transactionDataList = new ArrayList<>(this.transactions); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + long lastRequestTime = ArbitraryDataManager.getInstance().lastRequestForSignature(transactionData.getSignature()); + // If we haven't requested yet, or requested in the last 30 seconds, there's still a + // chance that data is on its way but hasn't arrived yet + if (lastRequestTime == 0 || now - lastRequestTime < 30 * 1000L) { + return true; + } + } + return false; + + } catch (DataException e) { + return false; + } + } + + + /** + * Best guess as to whether we are currently downloading a resource + * This is only used to give an indication to the user of progress + * @return - whether we are trying to download the resource + */ + private boolean isDownloading() { + try { + this.fetchTransactions(); + Long now = NTP.getTime(); + if (now == null) { + return false; + } + + List transactionDataList = new ArrayList<>(this.transactions); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + long lastRequestTime = ArbitraryDataManager.getInstance().lastRequestForSignature(transactionData.getSignature()); + // If were have requested data in the last 30 seconds, treat it as "downloading" + if (lastRequestTime > 0 && now - lastRequestTime < 30 * 1000L) { + return true; + } + } + + // FUTURE: we may want to check for file hashes (including the metadata file hash) in + // ArbitraryDataManager.arbitraryDataFileRequests and return true if one is found. + + return false; + + } catch (DataException e) { + return false; + } + } + + + + private void fetchTransactions() throws DataException { + if (this.transactions != null && !this.transactions.isEmpty()) { + // Already fetched + return; + } + + try (final Repository repository = RepositoryManager.getRepository()) { + + // Get the most recent PUT + ArbitraryTransactionData latestPut = repository.getArbitraryRepository() + .getLatestTransaction(this.resourceId, this.service, ArbitraryTransactionData.Method.PUT, this.identifier); + if (latestPut == null) { + String message = String.format("Couldn't find PUT transaction for name %s, service %s and identifier %s", + this.resourceId, this.service, this.identifierString()); + throw new DataException(message); + } + this.latestPutTransaction = latestPut; + + // Load all transactions since the latest PUT + List transactionDataList = repository.getArbitraryRepository() + .getArbitraryTransactions(this.resourceId, this.service, this.identifier, latestPut.getTimestamp()); + + this.transactions = transactionDataList; + this.layerCount = transactionDataList.size(); + } + } + + private String resourceIdString() { + return resourceId != null ? resourceId : ""; + } + + private String resourceIdTypeString() { + return resourceIdType != null ? resourceIdType.toString() : ""; + } + + private String serviceString() { + return service != null ? service.toString() : ""; + } + + private String identifierString() { + return identifier != null ? identifier : ""; + } + + @Override + public String toString() { + return String.format("%s %s %s", this.serviceString(), this.resourceIdString(), this.identifierString()); + } + + + /** + * @return unique key used to identify this resource + */ + public String getUniqueKey() { + return String.format("%s-%s-%s", this.service, this.resourceId, this.identifier).toLowerCase(); + } + + public String getResourceId() { + return this.resourceId; + } + + public Service getService() { + return this.service; + } + + public String getIdentifier() { + return this.identifier; + } +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java new file mode 100644 index 000000000..442461e16 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java @@ -0,0 +1,285 @@ +package org.qortal.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; +import org.qortal.arbitrary.ArbitraryDataDiff.*; +import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; +import org.qortal.arbitrary.misc.Service; +import org.qortal.block.BlockChain; +import org.qortal.crypto.Crypto; +import org.qortal.data.PaymentData; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.ArbitraryTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.transform.Transformer; +import org.qortal.utils.Base58; +import org.qortal.utils.FilesystemUtils; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +public class ArbitraryDataTransactionBuilder { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataTransactionBuilder.class); + + // Min transaction version required + private static final int MIN_TRANSACTION_VERSION = 5; + + // Maximum number of PATCH layers allowed + private static final int MAX_LAYERS = 10; + // Maximum size difference (out of 1) allowed for PATCH transactions + private static final double MAX_SIZE_DIFF = 0.2f; + // Maximum proportion of files modified relative to total + private static final double MAX_FILE_DIFF = 0.5f; + + private final String publicKey58; + private final Path path; + private final String name; + private Method method; + private final Service service; + private final String identifier; + private final Repository repository; + + private int chunkSize = ArbitraryDataFile.CHUNK_SIZE; + + private ArbitraryTransactionData arbitraryTransactionData; + private ArbitraryDataFile arbitraryDataFile; + + public ArbitraryDataTransactionBuilder(Repository repository, String publicKey58, Path path, String name, + Method method, Service service, String identifier) { + this.repository = repository; + this.publicKey58 = publicKey58; + this.path = path; + this.name = name; + this.method = method; + this.service = service; + + // If identifier is a blank string, or reserved keyword "default", treat it as null + if (identifier == null || identifier.equals("") || identifier.equals("default")) { + identifier = null; + } + this.identifier = identifier; + } + + public void build() throws DataException { + try { + this.preExecute(); + this.checkMethod(); + this.createTransaction(); + } + finally { + this.postExecute(); + } + } + + private void preExecute() { + + } + + private void postExecute() { + + } + + private void checkMethod() throws DataException { + if (this.method == null) { + // We need to automatically determine the method + this.method = this.determineMethodAutomatically(); + } + } + + private Method determineMethodAutomatically() throws DataException { + ArbitraryDataReader reader = new ArbitraryDataReader(this.name, ResourceIdType.NAME, this.service, this.identifier); + try { + reader.loadSynchronously(true); + } catch (Exception e) { + // Catch all exceptions if the existing resource cannot be loaded first time + // In these cases it's simplest to just use a PUT transaction + return Method.PUT; + } + + try { + // Check layer count + int layerCount = reader.getLayerCount(); + if (layerCount >= MAX_LAYERS) { + LOGGER.info("Reached maximum layer count ({} / {}) - using PUT", layerCount, MAX_LAYERS); + return Method.PUT; + } + + // Check size of differences between this layer and previous layer + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(reader.getFilePath(), this.path, reader.getLatestSignature()); + patch.create(); + long diffSize = FilesystemUtils.getDirectorySize(patch.getFinalPath()); + long existingStateSize = FilesystemUtils.getDirectorySize(reader.getFilePath()); + double difference = (double) diffSize / (double) existingStateSize; + if (difference > MAX_SIZE_DIFF) { + LOGGER.info("Reached maximum difference ({} / {}) - using PUT", difference, MAX_SIZE_DIFF); + return Method.PUT; + } + + // Check number of modified files + ArbitraryDataMetadataPatch metadata = patch.getMetadata(); + int totalFileCount = patch.getTotalFileCount(); + int differencesCount = metadata.getFileDifferencesCount(); + difference = (double) differencesCount / (double) totalFileCount; + if (difference > MAX_FILE_DIFF) { + LOGGER.info("Reached maximum file differences ({} / {}) - using PUT", difference, MAX_FILE_DIFF); + return Method.PUT; + } + + // Check the patch types + // Limit this check to single file resources only for now + boolean atLeastOnePatch = false; + if (totalFileCount == 1) { + for (ModifiedPath path : metadata.getModifiedPaths()) { + if (path.getDiffType() != DiffType.COMPLETE_FILE) { + atLeastOnePatch = true; + } + } + } + if (!atLeastOnePatch) { + LOGGER.info("Patch consists of complete files only - using PUT"); + return Method.PUT; + } + + // State is appropriate for a PATCH transaction + return Method.PATCH; + } + catch (IOException | DataException e) { + // Handle matching states separately, as it's best to block transactions with duplicate states + if (e.getMessage().equals("Current state matches previous state. Nothing to do.")) { + throw new DataException(e.getMessage()); + } + LOGGER.info("Caught exception: {}", e.getMessage()); + LOGGER.info("Unable to load existing resource - using PUT to overwrite it."); + return Method.PUT; + } + } + + private void createTransaction() throws DataException { + arbitraryDataFile = null; + try { + Long now = NTP.getTime(); + if (now == null) { + throw new DataException("NTP time not synced yet"); + } + + // Ensure that this chain supports transactions necessary for complex arbitrary data + int transactionVersion = Transaction.getVersionByTimestamp(now); + if (transactionVersion < MIN_TRANSACTION_VERSION) { + throw new DataException("Transaction version unsupported on this blockchain."); + } + + if (publicKey58 == null || path == null) { + throw new DataException("Missing public key or path"); + } + byte[] creatorPublicKey = Base58.decode(publicKey58); + final String creatorAddress = Crypto.toAddress(creatorPublicKey); + byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); + if (lastReference == null) { + // Use a random last reference on the very first transaction for an account + // Code copied from CrossChainResource.buildAtMessage() + // We already require PoW on all arbitrary transactions, so no additional logic is needed + Random random = new Random(); + lastReference = new byte[Transformer.SIGNATURE_LENGTH]; + random.nextBytes(lastReference); + } + + Compression compression = Compression.ZIP; + + // FUTURE? Use zip compression for directories, or no compression for single files + // Compression compression = (path.toFile().isDirectory()) ? Compression.ZIP : Compression.NONE; + + ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method, compression); + try { + arbitraryDataWriter.setChunkSize(this.chunkSize); + arbitraryDataWriter.save(); + } catch (IOException | DataException | InterruptedException | RuntimeException | MissingDataException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); + throw new DataException(e.getMessage()); + } + + // Get main file + arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile(); + if (arbitraryDataFile == null) { + throw new DataException("Arbitrary data file is null"); + } + + // Get chunks metadata file + ArbitraryDataFile metadataFile = arbitraryDataFile.getMetadataFile(); + if (metadataFile == null && arbitraryDataFile.chunkCount() > 1) { + throw new DataException(String.format("Chunks metadata data file is null but there are %d chunks", arbitraryDataFile.chunkCount())); + } + + String digest58 = arbitraryDataFile.digest58(); + if (digest58 == null) { + LOGGER.error("Unable to calculate file digest"); + throw new DataException("Unable to calculate file digest"); + } + + final BaseTransactionData baseTransactionData = new BaseTransactionData(now, Group.NO_GROUP, + lastReference, creatorPublicKey, 0L, null); + final int size = (int) arbitraryDataFile.size(); + final int version = 5; + final int nonce = 0; + byte[] secret = arbitraryDataFile.getSecret(); + final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; + final byte[] digest = arbitraryDataFile.digest(); + final byte[] metadataHash = (metadataFile != null) ? metadataFile.getHash() : null; + final List payments = new ArrayList<>(); + + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + version, service, nonce, size, name, identifier, method, + secret, compression, digest, dataType, metadataHash, payments); + + this.arbitraryTransactionData = transactionData; + + } catch (DataException e) { + if (arbitraryDataFile != null) { + arbitraryDataFile.deleteAll(); + } + throw(e); + } + + } + + public void computeNonce() throws DataException { + if (this.arbitraryTransactionData == null) { + throw new DataException("Arbitrary transaction data is required to compute nonce"); + } + + ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, this.arbitraryTransactionData); + LOGGER.info("Computing nonce..."); + transaction.computeNonce(); + + Transaction.ValidationResult result = transaction.isValidUnconfirmed(); + if (result != Transaction.ValidationResult.OK) { + arbitraryDataFile.deleteAll(); + throw new DataException(String.format("Arbitrary transaction invalid: %s", result)); + } + LOGGER.info("Transaction is valid"); + } + + public ArbitraryTransactionData getArbitraryTransactionData() { + return this.arbitraryTransactionData; + } + + public ArbitraryDataFile getArbitraryDataFile() { + return this.arbitraryDataFile; + } + + public void setChunkSize(int chunkSize) { + this.chunkSize = chunkSize; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java new file mode 100644 index 000000000..39ba4ade4 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -0,0 +1,342 @@ +package org.qortal.arbitrary; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; +import org.qortal.arbitrary.misc.Service; +import org.qortal.crypto.Crypto; +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.crypto.AES; +import org.qortal.repository.DataException; +import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.settings.Settings; +import org.qortal.utils.Base58; +import org.qortal.utils.FilesystemUtils; +import org.qortal.utils.ZipUtils; + +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +public class ArbitraryDataWriter { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataWriter.class); + + private Path filePath; + private final String name; + private final Service service; + private final String identifier; + private final Method method; + private final Compression compression; + + private int chunkSize = ArbitraryDataFile.CHUNK_SIZE; + + private SecretKey aesKey; + private ArbitraryDataFile arbitraryDataFile; + + // Intermediate paths to cleanup + private Path workingPath; + private Path compressedPath; + private Path encryptedPath; + + public ArbitraryDataWriter(Path filePath, String name, Service service, String identifier, Method method, Compression compression) { + this.filePath = filePath; + this.name = name; + this.service = service; + this.method = method; + this.compression = compression; + + // If identifier is a blank string, or reserved keyword "default", treat it as null + if (identifier == null || identifier.equals("") || identifier.equals("default")) { + identifier = null; + } + this.identifier = identifier; + } + + public void save() throws IOException, DataException, InterruptedException, MissingDataException { + try { + this.preExecute(); + this.validateService(); + this.process(); + this.compress(); + this.encrypt(); + this.split(); + this.createMetadataFile(); + this.validate(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() throws DataException { + this.checkEnabled(); + + // Enforce compression when uploading a directory + File file = new File(this.filePath.toString()); + if (file.isDirectory() && compression == Compression.NONE) { + throw new DataException("Unable to upload a directory without compression"); + } + + // Create temporary working directory + this.createWorkingDirectory(); + } + + private void postExecute() throws IOException { + this.cleanupFilesystem(); + } + + private void checkEnabled() throws DataException { + if (!Settings.getInstance().isQdnEnabled()) { + throw new DataException("QDN is disabled in settings"); + } + } + + private void createWorkingDirectory() throws DataException { + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + String identifier = Base58.encode(Crypto.digest(this.filePath.toString().getBytes())); + Path tempDir = Paths.get(baseDir, "writer", identifier); + try { + Files.createDirectories(tempDir); + } catch (IOException e) { + throw new DataException("Unable to create temp directory"); + } + this.workingPath = tempDir; + } + + private void validateService() throws IOException, DataException { + if (this.service.isValidationRequired()) { + Service.ValidationResult result = this.service.validate(this.filePath); + if (result != Service.ValidationResult.OK) { + throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString())); + } + } + } + + private void process() throws DataException, IOException, MissingDataException { + switch (this.method) { + + case PUT: + // Nothing to do + break; + + case PATCH: + this.processPatch(); + break; + + default: + throw new DataException(String.format("Unknown method specified: %s", method.toString())); + } + } + + private void processPatch() throws DataException, IOException, MissingDataException { + + // Build the existing state using past transactions + ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.name, this.service, this.identifier); + builder.build(); + Path builtPath = builder.getFinalPath(); + + // Obtain the latest signature, so this can be included in the patch + byte[] latestSignature = builder.getLatestSignature(); + + // Compute a diff of the latest changes on top of the previous state + // Then use only the differences as our data payload + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(builtPath, this.filePath, latestSignature); + patch.create(); + this.filePath = patch.getFinalPath(); + + // Delete the input directory + if (FilesystemUtils.pathInsideDataOrTempPath(builtPath)) { + File directory = new File(builtPath.toString()); + FileUtils.deleteDirectory(directory); + } + + // Validate the patch + this.validatePatch(); + } + + private void validatePatch() throws DataException { + if (this.filePath == null) { + throw new DataException("Null path after creating patch"); + } + + File qortalMetadataDirectoryFile = Paths.get(this.filePath.toString(), ".qortal").toFile(); + if (!qortalMetadataDirectoryFile.exists()) { + throw new DataException("Qortal metadata folder doesn't exist in patch"); + } + if (!qortalMetadataDirectoryFile.isDirectory()) { + throw new DataException("Qortal metadata folder isn't a directory"); + } + + File qortalPatchMetadataFile = Paths.get(this.filePath.toString(), ".qortal", "patch").toFile(); + if (!qortalPatchMetadataFile.exists()) { + throw new DataException("Qortal patch metadata file doesn't exist in patch"); + } + if (!qortalPatchMetadataFile.isFile()) { + throw new DataException("Qortal patch metadata file isn't a file"); + } + } + + private void compress() throws InterruptedException, DataException { + // Compress the data if requested + if (this.compression != Compression.NONE) { + this.compressedPath = Paths.get(this.workingPath.toString(), "data.zip"); + try { + + if (this.compression == Compression.ZIP) { + LOGGER.info("Compressing..."); + String enclosingFolderName = "data"; + ZipUtils.zip(this.filePath.toString(), this.compressedPath.toString(), enclosingFolderName); + } + else { + throw new DataException(String.format("Unknown compression type specified: %s", compression.toString())); + } + // FUTURE: other compression types + + // Delete the input directory + if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { + File directory = new File(this.filePath.toString()); + FileUtils.deleteDirectory(directory); + } + // Replace filePath pointer with the zipped file path + this.filePath = this.compressedPath; + + } catch (IOException | DataException e) { + throw new DataException("Unable to zip directory", e); + } + } + } + + private void encrypt() throws DataException { + this.encryptedPath = Paths.get(this.workingPath.toString(), "data.zip.encrypted"); + try { + // Encrypt the file with AES + LOGGER.info("Encrypting..."); + this.aesKey = AES.generateKey(256); + AES.encryptFile("AES/CBC/PKCS5Padding", this.aesKey, this.filePath.toString(), this.encryptedPath.toString()); + + // Delete the input file + if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { + Files.delete(this.filePath); + } + // Replace filePath pointer with the encrypted file path + this.filePath = this.encryptedPath; + + } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException + | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) { + throw new DataException(String.format("Unable to encrypt file %s: %s", this.filePath, e.getMessage())); + } + } + + private void split() throws IOException, DataException { + // We don't have a signature yet, so use null to put the file in a generic folder + this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath, null); + if (this.arbitraryDataFile == null) { + throw new IOException("No file available when trying to split"); + } + + int chunkCount = this.arbitraryDataFile.split(this.chunkSize); + if (chunkCount > 0) { + LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); + } + else { + throw new DataException("Unable to split file into chunks"); + } + } + + private void createMetadataFile() throws IOException, DataException { + // If we have at least one chunk, we need to create an index file containing their hashes + if (this.arbitraryDataFile.chunkCount() > 1) { + // Create the JSON file + Path chunkFilePath = Paths.get(this.workingPath.toString(), "metadata.json"); + ArbitraryDataTransactionMetadata chunkMetadata = new ArbitraryDataTransactionMetadata(chunkFilePath); + chunkMetadata.setChunks(this.arbitraryDataFile.chunkHashList()); + chunkMetadata.write(); + + // Create an ArbitraryDataFile from the JSON file (we don't have a signature yet) + ArbitraryDataFile metadataFile = ArbitraryDataFile.fromPath(chunkFilePath, null); + this.arbitraryDataFile.setMetadataFile(metadataFile); + } + } + + private void validate() throws IOException, DataException { + if (this.arbitraryDataFile == null) { + throw new DataException("No file available when validating"); + } + this.arbitraryDataFile.setSecret(this.aesKey.getEncoded()); + + // Validate the file + ValidationResult validationResult = this.arbitraryDataFile.isValid(); + if (validationResult != ValidationResult.OK) { + throw new DataException(String.format("File %s failed validation: %s", this.arbitraryDataFile, validationResult)); + } + LOGGER.info("Whole file hash is valid: {}", this.arbitraryDataFile.digest58()); + + // Validate each chunk + for (ArbitraryDataFileChunk chunk : this.arbitraryDataFile.getChunks()) { + validationResult = chunk.isValid(); + if (validationResult != ValidationResult.OK) { + throw new DataException(String.format("Chunk %s failed validation: %s", chunk, validationResult)); + } + } + LOGGER.info("Chunk hashes are valid"); + + // Validate chunks metadata file + if (this.arbitraryDataFile.chunkCount() > 1) { + ArbitraryDataFile metadataFile = this.arbitraryDataFile.getMetadataFile(); + if (metadataFile == null || !metadataFile.exists()) { + throw new DataException("No metadata file available, but there are multiple chunks"); + } + // Read the file + ArbitraryDataTransactionMetadata metadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath()); + metadata.read(); + // Check all chunks exist + for (byte[] chunk : this.arbitraryDataFile.chunkHashList()) { + if (!metadata.containsChunk(chunk)) { + throw new DataException(String.format("Missing chunk %s in metadata file", Base58.encode(chunk))); + } + } + } + } + + private void cleanupFilesystem() throws IOException { + // Clean up + if (FilesystemUtils.pathInsideDataOrTempPath(this.compressedPath)) { + File zippedFile = new File(this.compressedPath.toString()); + if (zippedFile.exists()) { + zippedFile.delete(); + } + } + if (FilesystemUtils.pathInsideDataOrTempPath(this.encryptedPath)) { + File encryptedFile = new File(this.encryptedPath.toString()); + if (encryptedFile.exists()) { + encryptedFile.delete(); + } + } + if (FilesystemUtils.pathInsideDataOrTempPath(this.workingPath)) { + FileUtils.deleteDirectory(new File(this.workingPath.toString())); + } + } + + + public ArbitraryDataFile getArbitraryDataFile() { + return this.arbitraryDataFile; + } + + public void setChunkSize(int chunkSize) { + this.chunkSize = chunkSize; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/exception/MissingDataException.java b/src/main/java/org/qortal/arbitrary/exception/MissingDataException.java new file mode 100644 index 000000000..63f617c0a --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/exception/MissingDataException.java @@ -0,0 +1,20 @@ +package org.qortal.arbitrary.exception; + +public class MissingDataException extends Exception { + + public MissingDataException() { + } + + public MissingDataException(String message) { + super(message); + } + + public MissingDataException(String message, Throwable cause) { + super(message, cause); + } + + public MissingDataException(Throwable cause) { + super(cause); + } + +} diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java new file mode 100644 index 000000000..127fefb55 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java @@ -0,0 +1,85 @@ +package org.qortal.arbitrary.metadata; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.repository.DataException; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * ArbitraryDataMetadata + * + * This is a base class to handle reading and writing JSON to the supplied filePath. + * + * It is not usable on its own; it must be subclassed, with two methods overridden: + * + * readJson() - code to unserialize the JSON file + * buildJson() - code to serialize the JSON file + * + */ +public class ArbitraryDataMetadata { + + protected static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadata.class); + + protected Path filePath; + + protected String jsonString; + + public ArbitraryDataMetadata(Path filePath) { + this.filePath = filePath; + } + + protected void readJson() throws DataException { + // To be overridden + } + + protected void buildJson() { + // To be overridden + } + + + public void read() throws IOException, DataException { + this.loadJson(); + this.readJson(); + } + + public void write() throws IOException, DataException { + this.buildJson(); + this.createParentDirectories(); + + BufferedWriter writer = new BufferedWriter(new FileWriter(this.filePath.toString())); + writer.write(this.jsonString); + writer.newLine(); + writer.close(); + } + + + protected void loadJson() throws IOException { + File metadataFile = new File(this.filePath.toString()); + if (!metadataFile.exists()) { + throw new IOException(String.format("Metadata file doesn't exist: %s", this.filePath.toString())); + } + + this.jsonString = new String(Files.readAllBytes(this.filePath)); + } + + + protected void createParentDirectories() throws DataException { + try { + Files.createDirectories(this.filePath.getParent()); + } catch (IOException e) { + throw new DataException("Unable to create parent directories"); + } + } + + + public String getJsonString() { + return this.jsonString; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java new file mode 100644 index 000000000..bd6bb2198 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java @@ -0,0 +1,69 @@ +package org.qortal.arbitrary.metadata; + +import org.json.JSONObject; +import org.qortal.repository.DataException; +import org.qortal.utils.Base58; + +import java.nio.file.Path; + +public class ArbitraryDataMetadataCache extends ArbitraryDataQortalMetadata { + + private byte[] signature; + private long timestamp; + + public ArbitraryDataMetadataCache(Path filePath) { + super(filePath); + + } + + @Override + protected String fileName() { + return "cache"; + } + + @Override + protected void readJson() throws DataException { + if (this.jsonString == null) { + throw new DataException("Patch JSON string is null"); + } + + JSONObject cache = new JSONObject(this.jsonString); + if (cache.has("signature")) { + String sig = cache.getString("signature"); + if (sig != null) { + this.signature = Base58.decode(sig); + } + } + if (cache.has("timestamp")) { + this.timestamp = cache.getLong("timestamp"); + } + } + + @Override + protected void buildJson() { + JSONObject patch = new JSONObject(); + patch.put("signature", Base58.encode(this.signature)); + patch.put("timestamp", this.timestamp); + + this.jsonString = patch.toString(2); + LOGGER.trace("Cache metadata: {}", this.jsonString); + } + + + public void setSignature(byte[] signature) { + this.signature = signature; + } + + public byte[] getSignature() { + return this.signature; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public long getTimestamp() { + return this.timestamp; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java new file mode 100644 index 000000000..954dcb03e --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java @@ -0,0 +1,182 @@ +package org.qortal.arbitrary.metadata; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; +import org.qortal.arbitrary.ArbitraryDataDiff.*; +import org.qortal.repository.DataException; +import org.qortal.utils.Base58; + +import java.lang.reflect.Field; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; + +public class ArbitraryDataMetadataPatch extends ArbitraryDataQortalMetadata { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadataPatch.class); + + private List addedPaths; + private List modifiedPaths; + private List removedPaths; + private byte[] previousSignature; + private byte[] previousHash; + private byte[] currentHash; + + public ArbitraryDataMetadataPatch(Path filePath) { + super(filePath); + + this.addedPaths = new ArrayList<>(); + this.modifiedPaths = new ArrayList<>(); + this.removedPaths = new ArrayList<>(); + } + + @Override + protected String fileName() { + return "patch"; + } + + @Override + protected void readJson() throws DataException { + if (this.jsonString == null) { + throw new DataException("Patch JSON string is null"); + } + + JSONObject patch = new JSONObject(this.jsonString); + if (patch.has("prevSig")) { + String prevSig = patch.getString("prevSig"); + if (prevSig != null) { + this.previousSignature = Base58.decode(prevSig); + } + } + if (patch.has("prevHash")) { + String prevHash = patch.getString("prevHash"); + if (prevHash != null) { + this.previousHash = Base58.decode(prevHash); + } + } + if (patch.has("curHash")) { + String curHash = patch.getString("curHash"); + if (curHash != null) { + this.currentHash = Base58.decode(curHash); + } + } + if (patch.has("added")) { + JSONArray added = (JSONArray) patch.get("added"); + if (added != null) { + for (int i=0; i()); + changeMap.setAccessible(false); + } catch (IllegalAccessException | NoSuchFieldException e) { + // Don't worry about failures as this is for optional ordering only + } + + patch.put("prevSig", Base58.encode(this.previousSignature)); + patch.put("prevHash", Base58.encode(this.previousHash)); + patch.put("curHash", Base58.encode(this.currentHash)); + patch.put("added", new JSONArray(this.addedPaths)); + patch.put("removed", new JSONArray(this.removedPaths)); + + JSONArray modifiedPaths = new JSONArray(); + for (ModifiedPath modifiedPath : this.modifiedPaths) { + JSONObject modifiedPathJson = new JSONObject(); + modifiedPathJson.put("path", modifiedPath.getPath()); + modifiedPathJson.put("type", modifiedPath.getDiffType()); + modifiedPaths.put(modifiedPathJson); + } + patch.put("modified", modifiedPaths); + + this.jsonString = patch.toString(2); + LOGGER.debug("Patch metadata: {}", this.jsonString); + } + + public void setAddedPaths(List addedPaths) { + this.addedPaths = addedPaths; + } + + public List getAddedPaths() { + return this.addedPaths; + } + + public void setModifiedPaths(List modifiedPaths) { + this.modifiedPaths = modifiedPaths; + } + + public List getModifiedPaths() { + return this.modifiedPaths; + } + + public void setRemovedPaths(List removedPaths) { + this.removedPaths = removedPaths; + } + + public List getRemovedPaths() { + return this.removedPaths; + } + + public void setPreviousSignature(byte[] previousSignature) { + this.previousSignature = previousSignature; + } + + public byte[] getPreviousSignature() { + return this.previousSignature; + } + + public void setPreviousHash(byte[] previousHash) { + this.previousHash = previousHash; + } + + public byte[] getPreviousHash() { + return this.previousHash; + } + + public void setCurrentHash(byte[] currentHash) { + this.currentHash = currentHash; + } + + public byte[] getCurrentHash() { + return this.currentHash; + } + + + public int getFileDifferencesCount() { + return this.addedPaths.size() + this.modifiedPaths.size() + this.removedPaths.size(); + } + +} diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java new file mode 100644 index 000000000..4c188843b --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java @@ -0,0 +1,102 @@ +package org.qortal.arbitrary.metadata; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.repository.DataException; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * ArbitraryDataQortalMetadata + * + * This is a base class to handle reading and writing JSON to a .qortal folder + * within the supplied filePath. This is used when storing data against an existing + * arbitrary data file structure. + * + * It is not usable on its own; it must be subclassed, with three methods overridden: + * + * fileName() - the file name to use within the .qortal folder + * readJson() - code to unserialize the JSON file + * buildJson() - code to serialize the JSON file + * + */ +public class ArbitraryDataQortalMetadata extends ArbitraryDataMetadata { + + protected static final Logger LOGGER = LogManager.getLogger(ArbitraryDataQortalMetadata.class); + + protected Path filePath; + protected Path qortalDirectoryPath; + + protected String jsonString; + + public ArbitraryDataQortalMetadata(Path filePath) { + super(filePath); + + this.qortalDirectoryPath = Paths.get(filePath.toString(), ".qortal"); + } + + protected String fileName() { + // To be overridden + return null; + } + + protected void readJson() throws DataException { + // To be overridden + } + + protected void buildJson() { + // To be overridden + } + + + @Override + public void read() throws IOException, DataException { + this.loadJson(); + this.readJson(); + } + + @Override + public void write() throws IOException, DataException { + this.buildJson(); + this.createParentDirectories(); + this.createQortalDirectory(); + + Path patchPath = Paths.get(this.qortalDirectoryPath.toString(), this.fileName()); + BufferedWriter writer = new BufferedWriter(new FileWriter(patchPath.toString())); + writer.write(this.jsonString); + writer.newLine(); + writer.close(); + } + + @Override + protected void loadJson() throws IOException { + Path path = Paths.get(this.qortalDirectoryPath.toString(), this.fileName()); + File patchFile = new File(path.toString()); + if (!patchFile.exists()) { + throw new IOException(String.format("Patch file doesn't exist: %s", path.toString())); + } + + this.jsonString = new String(Files.readAllBytes(path)); + } + + + protected void createQortalDirectory() throws DataException { + try { + Files.createDirectories(this.qortalDirectoryPath); + } catch (IOException e) { + throw new DataException("Unable to create .qortal directory"); + } + } + + + public String getJsonString() { + return this.jsonString; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java new file mode 100644 index 000000000..abd47ec9e --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java @@ -0,0 +1,78 @@ +package org.qortal.arbitrary.metadata; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.qortal.repository.DataException; +import org.qortal.utils.Base58; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata { + + private List chunks; + + public ArbitraryDataTransactionMetadata(Path filePath) { + super(filePath); + + } + + @Override + protected void readJson() throws DataException { + if (this.jsonString == null) { + throw new DataException("Transaction metadata JSON string is null"); + } + + List chunksList = new ArrayList<>(); + JSONObject cache = new JSONObject(this.jsonString); + if (cache.has("chunks")) { + JSONArray chunks = cache.getJSONArray("chunks"); + if (chunks != null) { + for (int i=0; i chunks) { + this.chunks = chunks; + } + + public List getChunks() { + return this.chunks; + } + + public boolean containsChunk(byte[] chunk) { + for (byte[] c : this.chunks) { + if (Arrays.equals(c, chunk)) { + return true; + } + } + return false; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java new file mode 100644 index 000000000..5d94d806d --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/misc/Service.java @@ -0,0 +1,131 @@ +package org.qortal.arbitrary.misc; + +import org.json.JSONObject; +import org.qortal.arbitrary.ArbitraryDataRenderer; +import org.qortal.transaction.Transaction; +import org.qortal.utils.FilesystemUtils; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +public enum Service { + AUTO_UPDATE(1, false, null, null), + ARBITRARY_DATA(100, false, null, null), + WEBSITE(200, true, null, null) { + @Override + public ValidationResult validate(Path path) { + // Custom validation function to require an index HTML file in the root directory + List fileNames = ArbitraryDataRenderer.indexFiles(); + String[] files = path.toFile().list(); + if (files != null) { + for (String file : files) { + Path fileName = Paths.get(file).getFileName(); + if (fileName != null && fileNames.contains(fileName.toString())) { + return ValidationResult.OK; + } + } + } + return ValidationResult.MISSING_INDEX_FILE; + } + }, + GIT_REPOSITORY(300, false, null, null), + IMAGE(400, true, 10*1024*1024L, null), + THUMBNAIL(410, true, 500*1024L, null), + VIDEO(500, false, null, null), + AUDIO(600, false, null, null), + BLOG(700, false, null, null), + BLOG_POST(777, false, null, null), + BLOG_COMMENT(778, false, null, null), + DOCUMENT(800, false, null, null), + LIST(900, true, null, null), + PLAYLIST(910, true, null, null), + APP(1000, false, null, null), + METADATA(1100, false, null, null), + QORTAL_METADATA(1111, true, 10*1024L, Arrays.asList("title", "description", "tags")); + + public final int value; + private final boolean requiresValidation; + private final Long maxSize; + private final List requiredKeys; + + private static final Map map = stream(Service.values()) + .collect(toMap(service -> service.value, service -> service)); + + Service(int value, boolean requiresValidation, Long maxSize, List requiredKeys) { + this.value = value; + this.requiresValidation = requiresValidation; + this.maxSize = maxSize; + this.requiredKeys = requiredKeys; + } + + public ValidationResult validate(Path path) throws IOException { + if (!this.isValidationRequired()) { + return ValidationResult.OK; + } + + byte[] data = FilesystemUtils.getSingleFileContents(path); + long size = FilesystemUtils.getDirectorySize(path); + + // Validate max size if needed + if (this.maxSize != null) { + if (size > this.maxSize) { + return ValidationResult.EXCEEDS_SIZE_LIMIT; + } + } + + // Validate required keys if needed + if (this.requiredKeys != null) { + if (data == null) { + return ValidationResult.MISSING_KEYS; + } + JSONObject json = Service.toJsonObject(data); + for (String key : this.requiredKeys) { + if (!json.has(key)) { + return ValidationResult.MISSING_KEYS; + } + } + } + + // Validation passed + return ValidationResult.OK; + } + + public boolean isValidationRequired() { + return this.requiresValidation; + } + + public static Service valueOf(int value) { + return map.get(value); + } + + public static JSONObject toJsonObject(byte[] data) { + String dataString = new String(data); + return new JSONObject(dataString); + } + + public enum ValidationResult { + OK(1), + MISSING_KEYS(2), + EXCEEDS_SIZE_LIMIT(3), + MISSING_INDEX_FILE(4); + + public final int value; + + private static final Map map = stream(Transaction.ValidationResult.values()).collect(toMap(result -> result.value, result -> result)); + + ValidationResult(int value) { + this.value = value; + } + + public static Transaction.ValidationResult valueOf(int value) { + return map.get(value); + } + } +} diff --git a/src/main/java/org/qortal/arbitrary/patch/UnifiedDiffPatch.java b/src/main/java/org/qortal/arbitrary/patch/UnifiedDiffPatch.java new file mode 100644 index 000000000..0408f4caf --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/patch/UnifiedDiffPatch.java @@ -0,0 +1,229 @@ +package org.qortal.arbitrary.patch; + +import com.github.difflib.DiffUtils; +import com.github.difflib.UnifiedDiffUtils; +import com.github.difflib.patch.Patch; +import com.github.difflib.patch.PatchFailedException; +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.crypto.Crypto; +import org.qortal.repository.DataException; +import org.qortal.settings.Settings; +import org.qortal.utils.FilesystemUtils; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +public class UnifiedDiffPatch { + + private static final Logger LOGGER = LogManager.getLogger(UnifiedDiffPatch.class); + + private final Path before; + private final Path after; + private final Path destination; + + private String identifier; + private Path validationPath; + + public UnifiedDiffPatch(Path before, Path after, Path destination) { + this.before = before; + this.after = after; + this.destination = destination; + } + + /** + * Create a patch based on the differences in path "after" + * compared with base path "before", outputting the patch + * to the "destination" path. + * + * @throws IOException + */ + public void create() throws IOException { + if (!Files.exists(before)) { + throw new IOException(String.format("File not found (before): %s", before.toString())); + } + if (!Files.exists(after)) { + throw new IOException(String.format("File not found (after): %s", after.toString())); + } + + // Ensure parent folders exist in the destination + File file = new File(destination.toString()); + File parent = file.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + + // Delete an existing file if it exists + File destFile = destination.toFile(); + if (destFile.exists() && destFile.isFile()) { + Files.delete(destination); + } + + // Load the two files into memory + List original = FileUtils.readLines(before.toFile(), StandardCharsets.UTF_8); + List revised = FileUtils.readLines(after.toFile(), StandardCharsets.UTF_8); + + // Check if the original file ends with a newline + boolean endsWithNewline = FilesystemUtils.fileEndsWithNewline(before); + + // Generate diff information + Patch diff = DiffUtils.diff(original, revised); + + // Generate unified diff format + String originalFileName = before.getFileName().toString(); + String revisedFileName = after.getFileName().toString(); + List unifiedDiff = UnifiedDiffUtils.generateUnifiedDiff(originalFileName, revisedFileName, original, diff, 0); + + // Write the diff to the destination directory + FileWriter fileWriter = new FileWriter(destination.toString(), true); + BufferedWriter writer = new BufferedWriter(fileWriter); + for (int i=0; i originalContents = FileUtils.readLines(originalPath.toFile(), StandardCharsets.UTF_8); + List patchContents = FileUtils.readLines(patchPath.toFile(), StandardCharsets.UTF_8); + + // Check if the patch file (and therefore the original file) ends with a newline + boolean endsWithNewline = FilesystemUtils.fileEndsWithNewline(patchPath); + + // At first, parse the unified diff file and get the patch + Patch patch = UnifiedDiffUtils.parseUnifiedDiff(patchContents); + + // Then apply the computed patch to the given text + try { + List patchedContents = DiffUtils.patch(originalContents, patch); + + // Write the patched file to the merge directory + FileWriter fileWriter = new FileWriter(mergePath.toString(), true); + BufferedWriter writer = new BufferedWriter(fileWriter); + for (int i=0; i + * Note that sleep-until-message support might set/reset + * sleep-related flags/values. + *

+ * {@link #getATStateData()} will return null if nothing happened. + *

+ * @param blockHeight + * @param blockTimestamp + * @return AT-generated transactions, possibly empty + * @throws DataException + */ public List run(int blockHeight, long blockTimestamp) throws DataException { String atAddress = this.atData.getATAddress(); QortalATAPI api = new QortalATAPI(repository, this.atData, blockTimestamp); QortalAtLoggerFactory loggerFactory = QortalAtLoggerFactory.getInstance(); - byte[] codeBytes = this.atData.getCodeBytes(); + if (!api.willExecute(blockHeight)) + // this.atStateData will be null + return Collections.emptyList(); // Fetch latest ATStateData for this AT ATStateData latestAtStateData = this.repository.getATRepository().getLatestATState(atAddress); @@ -99,30 +117,36 @@ public List run(int blockHeight, long blockTimestamp) throws Data throw new IllegalStateException("No previous AT state data found"); // [Re]create AT machine state using AT state data or from scratch as applicable + byte[] codeBytes = this.atData.getCodeBytes(); MachineState state = MachineState.fromBytes(api, loggerFactory, latestAtStateData.getStateData(), codeBytes); try { + api.preExecute(state); state.execute(); } catch (Exception e) { throw new DataException(String.format("Uncaught exception while running AT '%s'", atAddress), e); } - long creation = this.atData.getCreation(); byte[] stateData = state.toBytes(); byte[] stateHash = Crypto.digest(stateData); + + // Nothing happened? + if (state.getSteps() == 0 && Arrays.equals(stateHash, latestAtStateData.getStateHash())) + // We currently want to execute frozen ATs, to maintain backwards support. + if (state.isFrozen() == false) + // this.atStateData will be null + return Collections.emptyList(); + long atFees = api.calcFinalFees(state); + Long sleepUntilMessageTimestamp = this.atData.getSleepUntilMessageTimestamp(); - this.atStateData = new ATStateData(atAddress, blockHeight, creation, stateData, stateHash, atFees, false); + this.atStateData = new ATStateData(atAddress, blockHeight, stateData, stateHash, atFees, false, sleepUntilMessageTimestamp); return api.getTransactions(); } public void update(int blockHeight, long blockTimestamp) throws DataException { - // [Re]create AT machine state using AT state data or from scratch as applicable - QortalATAPI api = new QortalATAPI(repository, this.atData, blockTimestamp); - QortalAtLoggerFactory loggerFactory = QortalAtLoggerFactory.getInstance(); - - byte[] codeBytes = this.atData.getCodeBytes(); - MachineState state = MachineState.fromBytes(api, loggerFactory, this.atStateData.getStateData(), codeBytes); + // Extract minimal/flags-only AT machine state using AT state data + MachineState state = MachineState.flagsOnlyfromBytes(this.atStateData.getStateData()); // Save latest AT state data this.repository.getATRepository().save(this.atStateData); @@ -134,6 +158,10 @@ public void update(int blockHeight, long blockTimestamp) throws DataException { this.atData.setHadFatalError(state.hadFatalError()); this.atData.setIsFrozen(state.isFrozen()); this.atData.setFrozenBalance(state.getFrozenBalance()); + + // Special sleep-until-message support + this.atData.setSleepUntilMessageTimestamp(this.atStateData.getSleepUntilMessageTimestamp()); + this.repository.getATRepository().save(this.atData); } @@ -151,12 +179,8 @@ public void revert(int blockHeight, long blockTimestamp) throws DataException { if (previousStateData == null) throw new DataException("Can't find previous AT state data for " + atAddress); - // [Re]create AT machine state using AT state data or from scratch as applicable - QortalATAPI api = new QortalATAPI(repository, this.atData, blockTimestamp); - QortalAtLoggerFactory loggerFactory = QortalAtLoggerFactory.getInstance(); - - byte[] codeBytes = this.atData.getCodeBytes(); - MachineState state = MachineState.fromBytes(api, loggerFactory, previousStateData.getStateData(), codeBytes); + // Extract minimal/flags-only AT machine state using AT state data + MachineState state = MachineState.flagsOnlyfromBytes(previousStateData.getStateData()); // Update AT info in repository this.atData.setIsSleeping(state.isSleeping()); @@ -165,6 +189,10 @@ public void revert(int blockHeight, long blockTimestamp) throws DataException { this.atData.setHadFatalError(state.hadFatalError()); this.atData.setIsFrozen(state.isFrozen()); this.atData.setFrozenBalance(state.getFrozenBalance()); + + // Special sleep-until-message support + this.atData.setSleepUntilMessageTimestamp(previousStateData.getSleepUntilMessageTimestamp()); + this.repository.getATRepository().save(this.atData); } diff --git a/src/main/java/org/qortal/at/QortalATAPI.java b/src/main/java/org/qortal/at/QortalATAPI.java index bf7d2abc2..c393a684d 100644 --- a/src/main/java/org/qortal/at/QortalATAPI.java +++ b/src/main/java/org/qortal/at/QortalATAPI.java @@ -17,7 +17,6 @@ import org.qortal.account.NullAccount; import org.qortal.account.PublicKeyAccount; import org.qortal.asset.Asset; -import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.CiyamAtSettings; import org.qortal.crypto.Crypto; @@ -30,13 +29,14 @@ import org.qortal.data.transaction.PaymentTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.group.Group; -import org.qortal.repository.BlockRepository; +import org.qortal.repository.ATRepository; import org.qortal.repository.DataException; import org.qortal.repository.Repository; +import org.qortal.repository.ATRepository.NextTransactionInfo; import org.qortal.transaction.AtTransaction; -import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.utils.Base58; +import org.qortal.utils.BitTwiddling; import com.google.common.primitives.Bytes; @@ -75,8 +75,45 @@ public List getTransactions() { return this.transactions; } - public long calcFinalFees(MachineState state) { - return state.getSteps() * this.ciyamAtSettings.feePerStep; + public boolean willExecute(int blockHeight) throws DataException { + // Sleep-until-message/height checking + Long sleepUntilMessageTimestamp = this.atData.getSleepUntilMessageTimestamp(); + + if (sleepUntilMessageTimestamp != null) { + // Quicker to check height, if sleep-until-height also active + Integer sleepUntilHeight = this.atData.getSleepUntilHeight(); + + boolean wakeDueToHeight = sleepUntilHeight != null && sleepUntilHeight != 0 && blockHeight >= sleepUntilHeight; + + boolean wakeDueToMessage = false; + if (!wakeDueToHeight) { + // No avoiding asking repository + Timestamp previousTxTimestamp = new Timestamp(sleepUntilMessageTimestamp); + NextTransactionInfo nextTransactionInfo = this.repository.getATRepository().findNextTransaction(this.atData.getATAddress(), + previousTxTimestamp.blockHeight, + previousTxTimestamp.transactionSequence); + + wakeDueToMessage = nextTransactionInfo != null; + } + + // Can we skip? + if (!wakeDueToHeight && !wakeDueToMessage) + return false; + } + + return true; + } + + public void preExecute(MachineState state) { + // Sleep-until-message/height checking + Long sleepUntilMessageTimestamp = this.atData.getSleepUntilMessageTimestamp(); + + if (sleepUntilMessageTimestamp != null) { + // We've passed checks, so clear sleep-related flags/values + this.setIsSleeping(state, false); + this.setSleepUntilHeight(state, 0); + this.atData.setSleepUntilMessageTimestamp(null); + } } // Inherited methods from CIYAM AT API @@ -133,9 +170,9 @@ public void putPreviousBlockHashIntoA(MachineState state) { byte[] signature = blockSummaries.get(0).getSignature(); // Save some of minter's signature and transactions signature, so middle 24 bytes of the full 128 byte signature. - this.setA2(state, fromBytes(signature, 52)); - this.setA3(state, fromBytes(signature, 60)); - this.setA4(state, fromBytes(signature, 68)); + this.setA2(state, BitTwiddling.longFromBEBytes(signature, 52)); + this.setA3(state, BitTwiddling.longFromBEBytes(signature, 60)); + this.setA4(state, BitTwiddling.longFromBEBytes(signature, 68)); } catch (DataException e) { throw new RuntimeException("AT API unable to fetch previous block?", e); } @@ -147,61 +184,33 @@ public void putTransactionAfterTimestampIntoA(Timestamp timestamp, MachineState String atAddress = this.atData.getATAddress(); int height = timestamp.blockHeight; - int sequence = timestamp.transactionSequence + 1; + int sequence = timestamp.transactionSequence; - BlockRepository blockRepository = this.getRepository().getBlockRepository(); + if (state.getCurrentBlockHeight() < BlockChain.getInstance().getAtFindNextTransactionFixHeight()) + // Off-by-one bug still in effect + sequence += 1; + ATRepository.NextTransactionInfo nextTransactionInfo; try { - int currentHeight = blockRepository.getBlockchainHeight(); - List blockTransactions = null; - - while (height <= currentHeight) { - if (blockTransactions == null) { - BlockData blockData = blockRepository.fromHeight(height); - - if (blockData == null) - throw new DataException("Unable to fetch block " + height + " from repository?"); - - Block block = new Block(this.getRepository(), blockData); - - blockTransactions = block.getTransactions(); - } - - // No more transactions in this block? Try next block - if (sequence >= blockTransactions.size()) { - ++height; - sequence = 0; - blockTransactions = null; - continue; - } - - Transaction transaction = blockTransactions.get(sequence); - - // Transaction needs to be sent to specified recipient - List recipientAddresses = transaction.getRecipientAddresses(); - if (recipientAddresses.contains(atAddress)) { - // Found a transaction - - this.setA1(state, new Timestamp(height, timestamp.blockchainId, sequence).longValue()); + nextTransactionInfo = this.getRepository().getATRepository().findNextTransaction(atAddress, height, sequence); + } catch (DataException e) { + throw new RuntimeException("AT API unable to fetch next transaction?", e); + } - // Copy transaction's partial signature into the other three A fields for future verification that it's the same transaction - byte[] signature = transaction.getTransactionData().getSignature(); - this.setA2(state, fromBytes(signature, 8)); - this.setA3(state, fromBytes(signature, 16)); - this.setA4(state, fromBytes(signature, 24)); + if (nextTransactionInfo == null) { + // No more transactions for AT at this time - zero A and exit + this.zeroA(state); + return; + } - return; - } + // Found a transaction - // Transaction wasn't for us - keep going - ++sequence; - } + this.setA1(state, new Timestamp(nextTransactionInfo.height, timestamp.blockchainId, nextTransactionInfo.sequence).longValue()); - // No more transactions - zero A and exit - this.zeroA(state); - } catch (DataException e) { - throw new RuntimeException("AT API unable to fetch next transaction?", e); - } + // Copy transaction's partial signature into the other three A fields for future verification that it's the same transaction + this.setA2(state, BitTwiddling.longFromBEBytes(nextTransactionInfo.signature, 8)); + this.setA3(state, BitTwiddling.longFromBEBytes(nextTransactionInfo.signature, 16)); + this.setA4(state, BitTwiddling.longFromBEBytes(nextTransactionInfo.signature, 24)); } @Override @@ -282,7 +291,7 @@ public long generateRandomUsingTransactionInA(MachineState state) { byte[] hash = Crypto.digest(input); - return fromBytes(hash, 0); + return BitTwiddling.longFromBEBytes(hash, 0); } catch (DataException e) { throw new RuntimeException("AT API unable to fetch latest block from repository?", e); } @@ -296,30 +305,14 @@ public void putMessageFromTransactionInAIntoB(MachineState state) { TransactionData transactionData = this.getTransactionFromA(state); - byte[] messageData = null; - - switch (transactionData.getType()) { - case MESSAGE: - messageData = ((MessageTransactionData) transactionData).getData(); - break; - - case AT: - messageData = ((ATTransactionData) transactionData).getMessage(); - break; - - default: - return; - } - - // Check data length is appropriate, i.e. not larger than B - if (messageData.length > 4 * 8) - return; + byte[] messageData = this.getMessageFromTransaction(transactionData); // Pad messageData to fit B - byte[] paddedMessageData = Bytes.ensureCapacity(messageData, 4 * 8, 0); + if (messageData.length < 4 * 8) + messageData = Bytes.ensureCapacity(messageData, 4 * 8, 0); // Endian must be correct here so that (for example) a SHA256 message can be compared to one generated locally - this.setB(state, paddedMessageData); + this.setB(state, messageData); } @Override @@ -457,10 +450,8 @@ public void platformSpecificPostCheckExecute(FunctionData functionData, MachineS // Utility methods - /** Convert part of little-endian byte[] to long */ - /* package */ static long fromBytes(byte[] bytes, int start) { - return (bytes[start] & 0xffL) | (bytes[start + 1] & 0xffL) << 8 | (bytes[start + 2] & 0xffL) << 16 | (bytes[start + 3] & 0xffL) << 24 - | (bytes[start + 4] & 0xffL) << 32 | (bytes[start + 5] & 0xffL) << 40 | (bytes[start + 6] & 0xffL) << 48 | (bytes[start + 7] & 0xffL) << 56; + public long calcFinalFees(MachineState state) { + return state.getSteps() * this.ciyamAtSettings.feePerStep; } /** Returns partial transaction signature, used to verify we're operating on the same transaction and not naively using block height & sequence. */ @@ -473,7 +464,7 @@ private void verifyTransaction(TransactionData transactionData, MachineState sta // Compare end of transaction's signature against A2 thru A4 byte[] sig = transactionData.getSignature(); - if (this.getA2(state) != fromBytes(sig, 8) || this.getA3(state) != fromBytes(sig, 16) || this.getA4(state) != fromBytes(sig, 24)) + if (this.getA2(state) != BitTwiddling.longFromBEBytes(sig, 8) || this.getA3(state) != BitTwiddling.longFromBEBytes(sig, 16) || this.getA4(state) != BitTwiddling.longFromBEBytes(sig, 24)) throw new IllegalStateException("Transaction signature in A no longer matches signature from repository"); } @@ -497,6 +488,29 @@ private void verifyTransaction(TransactionData transactionData, MachineState sta } } + /** Returns message data from transaction. */ + /*package*/ byte[] getMessageFromTransaction(TransactionData transactionData) { + switch (transactionData.getType()) { + case MESSAGE: + return ((MessageTransactionData) transactionData).getData(); + + case AT: + return ((ATTransactionData) transactionData).getMessage(); + + default: + return null; + } + } + + /*package*/ void sleepUntilMessageOrHeight(MachineState state, long txTimestamp, Long sleepUntilHeight) { + this.setIsSleeping(state, true); + + this.atData.setSleepUntilMessageTimestamp(txTimestamp); + + if (sleepUntilHeight != null) + this.setSleepUntilHeight(state, sleepUntilHeight.intValue()); + } + /** Returns AT's account */ /* package */ Account getATAccount() { return new Account(this.repository, this.atData.getATAddress()); @@ -563,4 +577,8 @@ protected void setB(MachineState state, byte[] bBytes) { super.setB(state, bBytes); } + protected void zeroB(MachineState state) { + super.zeroB(state); + } + } diff --git a/src/main/java/org/qortal/at/QortalAtLogger.java b/src/main/java/org/qortal/at/QortalAtLogger.java index 703972a66..6dfdaefcf 100644 --- a/src/main/java/org/qortal/at/QortalAtLogger.java +++ b/src/main/java/org/qortal/at/QortalAtLogger.java @@ -691,6 +691,11 @@ public void error(final Supplier msgSupplier) { logger.logIfEnabled(FQCN, ERROR, null, msgSupplier, (Throwable) null); } + /** Java 8 version */ + public void error(final java.util.function.Supplier msgSupplier) { + logger.logIfEnabled(FQCN, ERROR, null, () -> msgSupplier.get(), (Throwable) null); + } + /** * Logs a message (only to be constructed if the logging level is the {@code ERROR} * level) including the stack trace of the {@link Throwable} t passed as parameter. @@ -1375,6 +1380,11 @@ public void debug(final Supplier msgSupplier) { logger.logIfEnabled(FQCN, DEBUG, null, msgSupplier, (Throwable) null); } + /** Java 8 version */ + public void debug(final java.util.function.Supplier msgSupplier) { + logger.logIfEnabled(FQCN, DEBUG, null, () -> msgSupplier.get(), (Throwable) null); + } + /** * Logs a message (only to be constructed if the logging level is the {@code DEBUG} * level) including the stack trace of the {@link Throwable} t passed as parameter. @@ -2059,6 +2069,11 @@ public void echo(final Supplier msgSupplier) { logger.logIfEnabled(FQCN, ECHO, null, msgSupplier, (Throwable) null); } + /** Java 8 version */ + public void echo(final java.util.function.Supplier msgSupplier) { + logger.logIfEnabled(FQCN, ECHO, null, () -> msgSupplier.get(), (Throwable) null); + } + /** * Logs a message (only to be constructed if the logging level is the {@code ECHO} * level) including the stack trace of the {@link Throwable} t passed as parameter. diff --git a/src/main/java/org/qortal/at/QortalFunctionCode.java b/src/main/java/org/qortal/at/QortalFunctionCode.java index cf6b1cfd8..7069290a4 100644 --- a/src/main/java/org/qortal/at/QortalFunctionCode.java +++ b/src/main/java/org/qortal/at/QortalFunctionCode.java @@ -10,8 +10,9 @@ import org.ciyam.at.FunctionData; import org.ciyam.at.IllegalFunctionCodeException; import org.ciyam.at.MachineState; -import org.qortal.crosschain.BTC; +import org.qortal.crosschain.Bitcoin; import org.qortal.crypto.Crypto; +import org.qortal.data.transaction.TransactionData; import org.qortal.settings.Settings; /** @@ -22,8 +23,107 @@ */ public enum QortalFunctionCode { /** - * 0x0510
- * Convert address in B to 20-byte value in LSB of B1, and all of B2 & B3. + * Returns length of message data from transaction in A.
+ * 0x0501
+ * If transaction has no 'message', returns -1. + */ + GET_MESSAGE_LENGTH_FROM_TX_IN_A(0x0501, 0, true) { + @Override + protected void postCheckExecute(FunctionData functionData, MachineState state, short rawFunctionCode) throws ExecutionException { + QortalATAPI api = (QortalATAPI) state.getAPI(); + + TransactionData transactionData = api.getTransactionFromA(state); + + byte[] messageData = api.getMessageFromTransaction(transactionData); + + if (messageData == null) + functionData.returnValue = -1L; + else + functionData.returnValue = (long) messageData.length; + } + }, + /** + * Put offset 'message' from transaction in A into B
+ * 0x0502 start-offset
+ * Copies up to 32 bytes of message data, starting at start-offset into B.
+ * If transaction has no 'message', or start-offset out of bounds, then zero B
+ * Example 'message' could be 256-bit shared secret + */ + PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B(0x0502, 1, false) { + @Override + protected void postCheckExecute(FunctionData functionData, MachineState state, short rawFunctionCode) throws ExecutionException { + QortalATAPI api = (QortalATAPI) state.getAPI(); + + // In case something goes wrong, or we don't have enough message data. + api.zeroB(state); + + if (functionData.value1 < 0 || functionData.value1 > Integer.MAX_VALUE) + return; + + int startOffset = functionData.value1.intValue(); + + TransactionData transactionData = api.getTransactionFromA(state); + + byte[] messageData = api.getMessageFromTransaction(transactionData); + + if (messageData == null || startOffset > messageData.length) + return; + + /* + * Copy up to 32 bytes of message data into B, + * retain order but pad with zeros in lower bytes. + * + * So a 4-byte message "a b c d" would copy thusly: + * a b c d 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + */ + int byteCount = Math.min(32, messageData.length - startOffset); + byte[] bBytes = new byte[32]; + + System.arraycopy(messageData, startOffset, bBytes, 0, byteCount); + + api.setB(state, bBytes); + } + }, + /** + * Sleep AT until a new message arrives after 'tx-timestamp'.
+ * 0x0503 tx-timestamp + */ + SLEEP_UNTIL_MESSAGE(0x0503, 1, false) { + @Override + protected void postCheckExecute(FunctionData functionData, MachineState state, short rawFunctionCode) throws ExecutionException { + if (functionData.value1 <= 0) + return; + + long txTimestamp = functionData.value1; + + QortalATAPI api = (QortalATAPI) state.getAPI(); + api.sleepUntilMessageOrHeight(state, txTimestamp, null); + } + }, + /** + * Sleep AT until a new message arrives, after 'tx-timestamp', or height reached.
+ * 0x0504 tx-timestamp height + */ + SLEEP_UNTIL_MESSAGE_OR_HEIGHT(0x0504, 2, false) { + @Override + protected void postCheckExecute(FunctionData functionData, MachineState state, short rawFunctionCode) throws ExecutionException { + if (functionData.value1 <= 0) + return; + + long txTimestamp = functionData.value1; + + if (functionData.value2 <= 0) + return; + + long sleepUntilHeight = functionData.value2; + + QortalATAPI api = (QortalATAPI) state.getAPI(); + api.sleepUntilMessageOrHeight(state, txTimestamp, sleepUntilHeight); + } + }, + /** + * Convert address in B to 20-byte value in LSB of B1, and all of B2 & B3.
+ * 0x0510 */ CONVERT_B_TO_PKH(0x0510, 0, false) { @Override @@ -38,21 +138,21 @@ protected void postCheckExecute(FunctionData functionData, MachineState state, s } }, /** - * 0x0511
* Convert 20-byte value in LSB of B1, and all of B2 & B3 to P2SH.
+ * 0x0511
* P2SH stored in lower 25 bytes of B. */ CONVERT_B_TO_P2SH(0x0511, 0, false) { @Override protected void postCheckExecute(FunctionData functionData, MachineState state, short rawFunctionCode) throws ExecutionException { - byte addressPrefix = Settings.getInstance().getBitcoinNet() == BTC.BitcoinNet.MAIN ? 0x05 : (byte) 0xc4; + byte addressPrefix = Settings.getInstance().getBitcoinNet() == Bitcoin.BitcoinNet.MAIN ? 0x05 : (byte) 0xc4; convertAddressInB(addressPrefix, state); } }, /** - * 0x0512
* Convert 20-byte value in LSB of B1, and all of B2 & B3 to Qortal address.
+ * 0x0512
* Qortal address stored in lower 25 bytes of B. */ CONVERT_B_TO_QORTAL(0x0512, 0, false) { diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index f17a07779..d6bb11f3f 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -6,6 +6,8 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; +import java.text.DecimalFormat; +import java.text.NumberFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -15,6 +17,7 @@ import java.util.Set; import java.util.stream.Collectors; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.account.Account; @@ -29,6 +32,7 @@ import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; +import org.qortal.data.account.EligibleQoraHolderData; import org.qortal.data.account.QortFromQoraData; import org.qortal.data.account.RewardShareData; import org.qortal.data.at.ATData; @@ -53,7 +57,6 @@ import org.qortal.utils.Amounts; import org.qortal.utils.Base58; import org.qortal.utils.NTP; -import org.roaringbitmap.IntIterator; import com.google.common.primitives.Bytes; import com.google.common.primitives.Longs; @@ -128,7 +131,7 @@ public static ValidationResult valueOf(int value) { @FunctionalInterface private interface BlockRewardDistributor { - long distribute(long amount, Map balanceChanges) throws DataException; + long distribute(long amount, Map balanceChanges) throws DataException; } /** Lazy-instantiated expanded info on block's online accounts. */ @@ -144,8 +147,8 @@ private static class ExpandedAccount { private final Account recipientAccount; private final AccountData recipientAccountData; - ExpandedAccount(Repository repository, int accountIndex) throws DataException { - this.rewardShareData = repository.getAccountRepository().getRewardShareByIndex(accountIndex); + ExpandedAccount(Repository repository, RewardShareData rewardShareData) throws DataException { + this.rewardShareData = rewardShareData; this.sharePercent = this.rewardShareData.getSharePercent(); this.mintingAccount = new Account(repository, this.rewardShareData.getMinter()); @@ -173,27 +176,34 @@ private static class ExpandedAccount { * * @return account-level share "bin" from blockchain config, or null if founder / none found */ - public AccountLevelShareBin getShareBin() { + public AccountLevelShareBin getShareBin(int blockHeight) { if (this.isMinterFounder) return null; final int accountLevel = this.mintingAccountData.getLevel(); if (accountLevel <= 0) - return null; + return null; // level 0 isn't included in any share bins - final AccountLevelShareBin[] shareBinsByLevel = BlockChain.getInstance().getShareBinsByAccountLevel(); + final BlockChain blockChain = BlockChain.getInstance(); + final AccountLevelShareBin[] shareBinsByLevel = blockChain.getShareBinsByAccountLevel(); if (accountLevel > shareBinsByLevel.length) return null; - return shareBinsByLevel[accountLevel]; + if (blockHeight < blockChain.getShareBinFixHeight()) + // Off-by-one bug still in effect + return shareBinsByLevel[accountLevel]; + + // level 1 stored at index 0, level 2 stored at index 1, etc. + return shareBinsByLevel[accountLevel-1]; + } - public long distribute(long accountAmount, Map balanceChanges) { + public long distribute(long accountAmount, Map balanceChanges) { if (this.isRecipientAlsoMinter) { // minter & recipient the same - simpler case LOGGER.trace(() -> String.format("Minter/recipient account %s share: %s", this.mintingAccount.getAddress(), Amounts.prettyAmount(accountAmount))); if (accountAmount != 0) - balanceChanges.merge(this.mintingAccount, accountAmount, Long::sum); + balanceChanges.merge(this.mintingAccount.getAddress(), accountAmount, Long::sum); } else { // minter & recipient different - extra work needed long recipientAmount = (accountAmount * this.sharePercent) / 100L / 100L; // because scaled by 2dp and 'percent' means "per 100" @@ -201,11 +211,11 @@ public long distribute(long accountAmount, Map balanceChanges) { LOGGER.trace(() -> String.format("Minter account %s share: %s", this.mintingAccount.getAddress(), Amounts.prettyAmount(minterAmount))); if (minterAmount != 0) - balanceChanges.merge(this.mintingAccount, minterAmount, Long::sum); + balanceChanges.merge(this.mintingAccount.getAddress(), minterAmount, Long::sum); LOGGER.trace(() -> String.format("Recipient account %s share: %s", this.recipientAccount.getAddress(), Amounts.prettyAmount(recipientAmount))); if (recipientAmount != 0) - balanceChanges.merge(this.recipientAccount, recipientAmount, Long::sum); + balanceChanges.merge(this.recipientAccount.getAddress(), recipientAmount, Long::sum); } // We always distribute all of the amount @@ -215,9 +225,14 @@ public long distribute(long accountAmount, Map balanceChanges) { /** Always use getExpandedAccounts() to access this, as it's lazy-instantiated. */ private List cachedExpandedAccounts = null; + /** Opportunistic cache of this block's valid online accounts. Only created by call to isValid(). */ + private List cachedValidOnlineAccounts = null; + /** Opportunistic cache of this block's valid online reward-shares. Only created by call to isValid(). */ + private List cachedOnlineRewardShares = null; + // Other useful constants - private static final BigInteger MAX_DISTANCE; + public static final BigInteger MAX_DISTANCE; static { byte[] maxValue = new byte[Transformer.PUBLIC_KEY_LENGTH]; Arrays.fill(maxValue, (byte) 0xFF); @@ -349,12 +364,8 @@ public static Block mint(Repository repository, BlockData parentBlockData, Priva System.arraycopy(onlineAccountData.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH); } - byte[] minterSignature; - try { - minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData.getMinterSignature(), minter, encodedOnlineAccounts)); - } catch (TransformationException e) { - throw new DataException("Unable to calculate next block minter signature", e); - } + byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData, + minter.getPublicKey(), encodedOnlineAccounts)); // Qortal: minter is always a reward-share, so find actual minter and get their effective minting level int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, minter.getPublicKey()); @@ -420,12 +431,8 @@ public Block remint(PrivateKeyAccount minter) throws DataException { int version = this.blockData.getVersion(); byte[] reference = this.blockData.getReference(); - byte[] minterSignature; - try { - minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData.getMinterSignature(), minter, this.blockData.getEncodedOnlineAccounts())); - } catch (TransformationException e) { - throw new DataException("Unable to calculate next block's minter signature", e); - } + byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData, + minter.getPublicKey(), this.blockData.getEncodedOnlineAccounts())); // Qortal: minter is always a reward-share, so find actual minter and get their effective minting level int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, minter.getPublicKey()); @@ -469,6 +476,16 @@ public PublicKeyAccount getMinter() { return this.minter; } + + public void setRepository(Repository repository) throws DataException { + this.repository = repository; + + for (Transaction transaction : this.getTransactions()) { + transaction.setRepository(repository); + } + } + + // More information /** @@ -517,8 +534,10 @@ public List getTransactions() throws DataException { long nonAtTransactionCount = transactionsData.stream().filter(transactionData -> transactionData.getType() != TransactionType.AT).count(); // The number of non-AT transactions fetched from repository should correspond with Block's transactionCount - if (nonAtTransactionCount != this.blockData.getTransactionCount()) + if (nonAtTransactionCount != this.blockData.getTransactionCount()) { + LOGGER.error(() -> String.format("Block's transactions from repository (%d) do not match block's transaction count (%d)", nonAtTransactionCount, this.blockData.getTransactionCount())); throw new IllegalStateException("Block's transactions from repository do not match block's transaction count"); + } this.transactions = new ArrayList<>(); @@ -564,23 +583,29 @@ public List getATStates() throws DataException { /** * Return expanded info on block's online accounts. *

+ * Typically called as part of Block.process() or Block.orphan() + * so ideally after any calls to Block.isValid(). + * * @throws DataException */ public List getExpandedAccounts() throws DataException { if (this.cachedExpandedAccounts != null) return this.cachedExpandedAccounts; - ConciseSet accountIndexes = BlockTransformer.decodeOnlineAccounts(this.blockData.getEncodedOnlineAccounts()); - List expandedAccounts = new ArrayList<>(); - - IntIterator iterator = accountIndexes.iterator(); - while (iterator.hasNext()) { - int accountIndex = iterator.next(); + // We might already have a cache of online, reward-shares thanks to isValid() + if (this.cachedOnlineRewardShares == null) { + ConciseSet accountIndexes = BlockTransformer.decodeOnlineAccounts(this.blockData.getEncodedOnlineAccounts()); + this.cachedOnlineRewardShares = repository.getAccountRepository().getRewardSharesByIndexes(accountIndexes.toArray()); - ExpandedAccount accountInfo = new ExpandedAccount(repository, accountIndex); - expandedAccounts.add(accountInfo); + if (this.cachedOnlineRewardShares == null) + throw new DataException("Online accounts invalid?"); } + List expandedAccounts = new ArrayList<>(); + + for (RewardShareData rewardShare : this.cachedOnlineRewardShares) + expandedAccounts.add(new ExpandedAccount(repository, rewardShare)); + this.cachedExpandedAccounts = expandedAccounts; return this.cachedExpandedAccounts; @@ -732,11 +757,7 @@ protected void calcMinterSignature() { if (!(this.minter instanceof PrivateKeyAccount)) throw new IllegalStateException("Block's minter is not a PrivateKeyAccount - can't sign!"); - try { - this.blockData.setMinterSignature(((PrivateKeyAccount) this.minter).sign(BlockTransformer.getBytesForMinterSignature(this.blockData))); - } catch (TransformationException e) { - throw new RuntimeException("Unable to calculate block's minter signature", e); - } + this.blockData.setMinterSignature(((PrivateKeyAccount) this.minter).sign(BlockTransformer.getBytesForMinterSignature(this.blockData))); } /** @@ -780,16 +801,49 @@ public static BigInteger calcBlockWeight(int parentHeight, byte[] parentBlockSig return BigInteger.valueOf(blockSummaryData.getOnlineAccountsCount()).shiftLeft(ACCOUNTS_COUNT_SHIFT).add(keyDistance); } - public static BigInteger calcChainWeight(int commonBlockHeight, byte[] commonBlockSignature, List blockSummaries) { + public static BigInteger calcChainWeight(int commonBlockHeight, byte[] commonBlockSignature, List blockSummaries, int maxHeight) { BigInteger cumulativeWeight = BigInteger.ZERO; int parentHeight = commonBlockHeight; byte[] parentBlockSignature = commonBlockSignature; + NumberFormat formatter = new DecimalFormat("0.###E0"); + boolean isLogging = LOGGER.getLevel().isLessSpecificThan(Level.TRACE); + int blockCount = 0; for (BlockSummaryData blockSummaryData : blockSummaries) { - cumulativeWeight = cumulativeWeight.shiftLeft(CHAIN_WEIGHT_SHIFT).add(calcBlockWeight(parentHeight, parentBlockSignature, blockSummaryData)); + blockCount++; + StringBuilder stringBuilder = isLogging ? new StringBuilder(512) : null; + + if (isLogging) + stringBuilder.append(formatter.format(cumulativeWeight)).append(" -> "); + + cumulativeWeight = cumulativeWeight.shiftLeft(CHAIN_WEIGHT_SHIFT); + if (isLogging) + stringBuilder.append(formatter.format(cumulativeWeight)).append(" + "); + + BigInteger blockWeight = calcBlockWeight(parentHeight, parentBlockSignature, blockSummaryData); + if (isLogging) + stringBuilder.append("(height: ") + .append(parentHeight + 1) + .append(", online: ") + .append(blockSummaryData.getOnlineAccountsCount()) + .append(") ") + .append(formatter.format(blockWeight)); + + cumulativeWeight = cumulativeWeight.add(blockWeight); + if (isLogging) + stringBuilder.append(" -> ").append(formatter.format(cumulativeWeight)); + + if (isLogging && blockSummaries.size() > 1) + LOGGER.debug(() -> stringBuilder.toString()); //NOSONAR S1612 (false positive?) + parentHeight = blockSummaryData.getHeight(); parentBlockSignature = blockSummaryData.getSignature(); + + // After this timestamp, we only compare the same number of blocks + if (NTP.getTime() >= BlockChain.getInstance().getCalcChainWeightTimestamp() && parentHeight >= maxHeight) + break; } + LOGGER.trace(String.format("Chain weight calculation was based on %d blocks", blockCount)); return cumulativeWeight; } @@ -914,19 +968,9 @@ public ValidationResult areOnlineAccountsValid() throws DataException { if (accountIndexes.size() != this.blockData.getOnlineAccountsCount()) return ValidationResult.ONLINE_ACCOUNTS_INVALID; - List expandedAccounts = new ArrayList<>(); - - IntIterator iterator = accountIndexes.iterator(); - while (iterator.hasNext()) { - int accountIndex = iterator.next(); - RewardShareData rewardShareData = repository.getAccountRepository().getRewardShareByIndex(accountIndex); - - // Check that claimed online account actually exists - if (rewardShareData == null) - return ValidationResult.ONLINE_ACCOUNT_UNKNOWN; - - expandedAccounts.add(rewardShareData); - } + List onlineRewardShares = repository.getAccountRepository().getRewardSharesByIndexes(accountIndexes.toArray()); + if (onlineRewardShares == null) + return ValidationResult.ONLINE_ACCOUNT_UNKNOWN; // If block is past a certain age then we simply assume the signatures were correct long signatureRequirementThreshold = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMinLifetime(); @@ -936,28 +980,51 @@ public ValidationResult areOnlineAccountsValid() throws DataException { if (this.blockData.getOnlineAccountsSignatures() == null || this.blockData.getOnlineAccountsSignatures().length == 0) return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MISSING; - if (this.blockData.getOnlineAccountsSignatures().length != expandedAccounts.size() * Transformer.SIGNATURE_LENGTH) + if (this.blockData.getOnlineAccountsSignatures().length != onlineRewardShares.size() * Transformer.SIGNATURE_LENGTH) return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED; // Check signatures - List onlineAccountsSignatures = BlockTransformer.decodeTimestampSignatures(this.blockData.getOnlineAccountsSignatures()); long onlineTimestamp = this.blockData.getOnlineAccountsTimestamp(); byte[] onlineTimestampBytes = Longs.toByteArray(onlineTimestamp); - List onlineAccounts = Controller.getInstance().getOnlineAccounts(); + + // If this block is much older than current online timestamp, then there's no point checking current online accounts + List currentOnlineAccounts = onlineTimestamp < NTP.getTime() - Controller.ONLINE_TIMESTAMP_MODULUS + ? null + : Controller.getInstance().getOnlineAccounts(); + List latestBlocksOnlineAccounts = Controller.getInstance().getLatestBlocksOnlineAccounts(); + + // Extract online accounts' timestamp signatures from block data + List onlineAccountsSignatures = BlockTransformer.decodeTimestampSignatures(this.blockData.getOnlineAccountsSignatures()); + + // We'll build up a list of online accounts to hand over to Controller if block is added to chain + // and this will become latestBlocksOnlineAccounts (above) to reduce CPU load when we process next block... + List ourOnlineAccounts = new ArrayList<>(); for (int i = 0; i < onlineAccountsSignatures.size(); ++i) { byte[] signature = onlineAccountsSignatures.get(i); - byte[] publicKey = expandedAccounts.get(i).getRewardSharePublicKey(); + byte[] publicKey = onlineRewardShares.get(i).getRewardSharePublicKey(); - // If signature is still current then no need to perform Ed25519 verify OnlineAccountData onlineAccountData = new OnlineAccountData(onlineTimestamp, signature, publicKey); - if (onlineAccounts.remove(onlineAccountData)) // remove() is like contains() but also reduces the number to check next time + ourOnlineAccounts.add(onlineAccountData); + + // If signature is still current then no need to perform Ed25519 verify + if (currentOnlineAccounts != null && currentOnlineAccounts.remove(onlineAccountData)) + // remove() returned true, so online account still current + // and one less entry in currentOnlineAccounts to check next time + continue; + + // If signature was okay in latest block then no need to perform Ed25519 verify + if (latestBlocksOnlineAccounts != null && latestBlocksOnlineAccounts.contains(onlineAccountData)) continue; if (!Crypto.verify(publicKey, signature, onlineTimestampBytes)) return ValidationResult.ONLINE_ACCOUNT_SIGNATURE_INCORRECT; } + // All online accounts valid, so save our list of online accounts for potential later use + this.cachedValidOnlineAccounts = ourOnlineAccounts; + this.cachedOnlineRewardShares = onlineRewardShares; + return ValidationResult.OK; } @@ -1037,6 +1104,15 @@ private ValidationResult areTransactionsValid() throws DataException { // Create repository savepoint here so we can rollback to it after testing transactions repository.setSavepoint(); + if (this.blockData.getHeight() == 212937) { + // Apply fix for block 212937 but fix will be rolled back before we exit method + Block212937.processFix(this); + } + else if (InvalidNameRegistrationBlocks.isAffectedBlock(this.blockData.getHeight())) { + // Apply fix for affected name registration blocks, but fix will be rolled back before we exit method + InvalidNameRegistrationBlocks.processFix(this); + } + for (Transaction transaction : this.getTransactions()) { TransactionData transactionData = transaction.getTransactionData(); @@ -1074,7 +1150,7 @@ private ValidationResult areTransactionsValid() throws DataException { // Check transaction can even be processed validationResult = transaction.isProcessable(); if (validationResult != Transaction.ValidationResult.OK) { - LOGGER.debug(String.format("Error during transaction validation, tx %s: %s", Base58.encode(transactionData.getSignature()), validationResult.name())); + LOGGER.info(String.format("Error during transaction validation, tx %s: %s", Base58.encode(transactionData.getSignature()), validationResult.name())); return ValidationResult.TRANSACTION_INVALID; } @@ -1188,12 +1264,13 @@ private void executeATs() throws DataException { for (ATData atData : executableATs) { AT at = new AT(this.repository, atData); List atTransactions = at.run(this.blockData.getHeight(), this.blockData.getTimestamp()); + ATStateData atStateData = at.getATStateData(); + // Didn't execute? (e.g. sleeping) + if (atStateData == null) + continue; allAtTransactions.addAll(atTransactions); - - ATStateData atStateData = at.getATStateData(); this.ourAtStates.add(atStateData); - this.ourAtFees += atStateData.getFees(); } @@ -1222,6 +1299,21 @@ protected boolean isMinterValid(Block parentBlock) throws DataException { return mintingAccount.canMint(); } + /** + * Pre-process block, and its transactions. + * This allows for any database integrity checks prior to validation. + * This is called before isValid() and process() + * + * @throws DataException + */ + public void preProcess() throws DataException { + List blocksTransactions = this.getTransactions(); + + for (Transaction transaction : blocksTransactions) { + transaction.preProcess(); + } + } + /** * Process block, and its transactions, adding them to the blockchain. * @@ -1240,6 +1332,10 @@ public void process() throws DataException { // Distribute block rewards, including transaction fees, before transactions processed processBlockRewards(); + + if (this.blockData.getHeight() == 212937) + // Apply fix for block 212937 + Block212937.processFix(this); } // We're about to (test-)process a batch of transactions, @@ -1271,6 +1367,12 @@ public void process() throws DataException { linkTransactionsToBlock(); postBlockTidy(); + + // Give Controller our cached, valid online accounts data (if any) to help reduce CPU load for next block + Controller.getInstance().pushLatestBlocksOnlineAccounts(this.cachedValidOnlineAccounts); + + // Log some debugging info relating to the block weight calculation + this.logDebugInfo(); } protected void increaseAccountLevels() throws DataException { @@ -1288,13 +1390,16 @@ protected void increaseAccountLevels() throws DataException { allUniqueExpandedAccounts.add(expandedAccount.recipientAccountData); } - // Decrease blocks minted count for all accounts + // Increase blocks minted count for all accounts + + // Batch update in repository + repository.getAccountRepository().modifyMintedBlockCounts(allUniqueExpandedAccounts.stream().map(AccountData::getAddress).collect(Collectors.toList()), +1); + + // Local changes and also checks for level bump for (AccountData accountData : allUniqueExpandedAccounts) { // Adjust count locally (in Java) accountData.setBlocksMinted(accountData.getBlocksMinted() + 1); - - int rowCount = repository.getAccountRepository().modifyMintedBlockCount(accountData.getAddress(), +1); - LOGGER.trace(() -> String.format("Block minter %s up to %d minted block%s (rowCount: %d)", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""), rowCount)); + LOGGER.trace(() -> String.format("Block minter %s up to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""))); final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment(); @@ -1449,6 +1554,9 @@ protected void linkTransactionsToBlock() throws DataException { public void orphan() throws DataException { LOGGER.trace(() -> String.format("Orphaning block %d", this.blockData.getHeight())); + // Log some debugging info relating to the block weight calculation + this.logDebugInfo(); + // Return AT fees and delete AT states from repository orphanAtFeesAndStates(); @@ -1462,6 +1570,10 @@ public void orphan() throws DataException { // Invalidate expandedAccounts as they may have changed due to orphaning TRANSFER_PRIVS transactions, etc. this.cachedExpandedAccounts = null; + if (this.blockData.getHeight() == 212937) + // Revert fix for block 212937 + Block212937.orphanFix(this); + // Block rewards, including transaction fees, removed after transactions undone orphanBlockRewards(); @@ -1474,6 +1586,9 @@ public void orphan() throws DataException { this.blockData.setHeight(null); postBlockTidy(); + + // Remove any cached, valid online accounts data from Controller + Controller.getInstance().popLatestBlocksOnlineAccounts(); } protected void orphanTransactionsFromBlock() throws DataException { @@ -1581,12 +1696,14 @@ protected void decreaseAccountLevels() throws DataException { } // Decrease blocks minted count for all accounts + + // Batch update in repository + repository.getAccountRepository().modifyMintedBlockCounts(allUniqueExpandedAccounts.stream().map(AccountData::getAddress).collect(Collectors.toList()), -1); + for (AccountData accountData : allUniqueExpandedAccounts) { // Adjust count locally (in Java) accountData.setBlocksMinted(accountData.getBlocksMinted() - 1); - - int rowCount = repository.getAccountRepository().modifyMintedBlockCount(accountData.getAddress(), -1); - LOGGER.trace(() -> String.format("Block minter %s down to %d minted block%s (rowCount: %d)", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""), rowCount)); + LOGGER.trace(() -> String.format("Block minter %s down to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""))); final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment(); @@ -1615,7 +1732,7 @@ public BlockRewardCandidate(String description, long share, BlockRewardDistribut this.distributionMethod = distributionMethod; } - public long distribute(long distibutionAmount, Map balanceChanges) throws DataException { + public long distribute(long distibutionAmount, Map balanceChanges) throws DataException { return this.distributionMethod.distribute(distibutionAmount, balanceChanges); } } @@ -1632,7 +1749,7 @@ protected void distributeBlockReward(long totalAmount) throws DataException { // Now distribute to candidates // Collate all balance changes and then apply in one final step - Map balanceChanges = new HashMap<>(); + Map balanceChanges = new HashMap<>(); long remainingAmount = totalAmount; for (int r = 0; r < rewardCandidates.size(); ++r) { @@ -1657,8 +1774,10 @@ protected void distributeBlockReward(long totalAmount) throws DataException { } // Apply balance changes - for (Map.Entry balanceChange : balanceChanges.entrySet()) - balanceChange.getKey().modifyAssetBalance(Asset.QORT, balanceChange.getValue()); + List accountBalanceDeltas = balanceChanges.entrySet().stream() + .map(entry -> new AccountBalanceData(entry.getKey(), Asset.QORT, entry.getValue())) + .collect(Collectors.toList()); + this.repository.getAccountRepository().modifyAssetBalances(accountBalanceDeltas); } protected List determineBlockRewardCandidates(boolean isProcessingNotOrphaning) throws DataException { @@ -1712,7 +1831,7 @@ protected List determineBlockRewardCandidates(boolean isPr // Find all accounts in share bin. getShareBin() returns null for minter accounts that are also founders, so they are effectively filtered out. AccountLevelShareBin accountLevelShareBin = accountLevelShareBins.get(binIndex); // Object reference compare is OK as all references are read-only from blockchain config. - List binnedAccounts = expandedAccounts.stream().filter(accountInfo -> accountInfo.getShareBin() == accountLevelShareBin).collect(Collectors.toList()); + List binnedAccounts = expandedAccounts.stream().filter(accountInfo -> accountInfo.getShareBin(this.blockData.getHeight()) == accountLevelShareBin).collect(Collectors.toList()); // No online accounts in this bin? Skip to next one if (binnedAccounts.isEmpty()) @@ -1728,7 +1847,7 @@ protected List determineBlockRewardCandidates(boolean isPr } // Fetch list of legacy QORA holders who haven't reached their cap of QORT reward. - List qoraHolders = this.repository.getAccountRepository().getEligibleLegacyQoraHolders(isProcessingNotOrphaning ? null : this.blockData.getHeight()); + List qoraHolders = this.repository.getAccountRepository().getEligibleLegacyQoraHolders(isProcessingNotOrphaning ? null : this.blockData.getHeight()); final boolean haveQoraHolders = !qoraHolders.isEmpty(); final long qoraHoldersShare = BlockChain.getInstance().getQoraHoldersShare(); @@ -1781,7 +1900,7 @@ protected List determineBlockRewardCandidates(boolean isPr return rewardCandidates; } - private static long distributeBlockRewardShare(long distributionAmount, List accounts, Map balanceChanges) { + private static long distributeBlockRewardShare(long distributionAmount, List accounts, Map balanceChanges) { // Collate all expanded accounts by minting account Map> accountsByMinter = new HashMap<>(); @@ -1810,7 +1929,7 @@ private static long distributeBlockRewardShare(long distributionAmount, List qoraHolders, Map balanceChanges, Block block) throws DataException { + private static long distributeBlockRewardToQoraHolders(long qoraHoldersAmount, List qoraHolders, Map balanceChanges, Block block) throws DataException { final boolean isProcessingNotOrphaning = qoraHoldersAmount >= 0; long qoraPerQortReward = BlockChain.getInstance().getQoraPerQortReward(); @@ -1818,7 +1937,7 @@ private static long distributeBlockRewardToQoraHolders(long qoraHoldersAmount, L long totalQoraHeld = 0; for (int i = 0; i < qoraHolders.size(); ++i) - totalQoraHeld += qoraHolders.get(i).getBalance(); + totalQoraHeld += qoraHolders.get(i).getQoraBalance(); long finalTotalQoraHeld = totalQoraHeld; LOGGER.trace(() -> String.format("Total legacy QORA held: %s", Amounts.prettyAmount(finalTotalQoraHeld))); @@ -1831,9 +1950,13 @@ private static long distributeBlockRewardToQoraHolders(long qoraHoldersAmount, L BigInteger totalQoraHeldBI = BigInteger.valueOf(totalQoraHeld); long sharedAmount = 0; + // For batched update of QORT_FROM_QORA balances + List newQortFromQoraBalances = new ArrayList<>(); + for (int h = 0; h < qoraHolders.size(); ++h) { - AccountBalanceData qoraHolder = qoraHolders.get(h); - BigInteger qoraHolderBalanceBI = BigInteger.valueOf(qoraHolder.getBalance()); + EligibleQoraHolderData qoraHolder = qoraHolders.get(h); + BigInteger qoraHolderBalanceBI = BigInteger.valueOf(qoraHolder.getQoraBalance()); + String qoraHolderAddress = qoraHolder.getAddress(); // This is where a 128bit integer library could help: // long holderReward = (qoraHoldersAmount * qoraHolder.getBalance()) / totalQoraHeld; @@ -1841,15 +1964,13 @@ private static long distributeBlockRewardToQoraHolders(long qoraHoldersAmount, L final long holderRewardForLogging = holderReward; LOGGER.trace(() -> String.format("QORA holder %s has %s / %s QORA so share: %s", - qoraHolder.getAddress(), Amounts.prettyAmount(qoraHolder.getBalance()), finalTotalQoraHeld, Amounts.prettyAmount(holderRewardForLogging))); + qoraHolderAddress, Amounts.prettyAmount(qoraHolder.getQoraBalance()), finalTotalQoraHeld, Amounts.prettyAmount(holderRewardForLogging))); // Too small to register this time? if (holderReward == 0) continue; - Account qoraHolderAccount = new Account(block.repository, qoraHolder.getAddress()); - - long newQortFromQoraBalance = qoraHolderAccount.getConfirmedBalance(Asset.QORT_FROM_QORA) + holderReward; + long newQortFromQoraBalance = qoraHolder.getQortFromQoraBalance() + holderReward; // If processing, make sure we don't overpay if (isProcessingNotOrphaning) { @@ -1863,44 +1984,43 @@ private static long distributeBlockRewardToQoraHolders(long qoraHoldersAmount, L newQortFromQoraBalance -= adjustment; // This is also the QORA holder's final QORT-from-QORA block - QortFromQoraData qortFromQoraData = new QortFromQoraData(qoraHolder.getAddress(), holderReward, block.blockData.getHeight()); + QortFromQoraData qortFromQoraData = new QortFromQoraData(qoraHolderAddress, holderReward, block.blockData.getHeight()); block.repository.getAccountRepository().save(qortFromQoraData); long finalAdjustedHolderReward = holderReward; LOGGER.trace(() -> String.format("QORA holder %s final share %s at height %d", - qoraHolder.getAddress(), Amounts.prettyAmount(finalAdjustedHolderReward), block.blockData.getHeight())); + qoraHolderAddress, Amounts.prettyAmount(finalAdjustedHolderReward), block.blockData.getHeight())); } } else { // Orphaning - QortFromQoraData qortFromQoraData = block.repository.getAccountRepository().getQortFromQoraInfo(qoraHolder.getAddress()); - if (qortFromQoraData != null) { + if (qoraHolder.getFinalBlockHeight() != null) { // Final QORT-from-QORA amount from repository was stored during processing, and hence positive. // So we use + here as qortFromQora is negative during orphaning. // More efficient than "holderReward - (0 - final-qort-from-qora)" - long adjustment = holderReward + qortFromQoraData.getFinalQortFromQora(); + long adjustment = holderReward + qoraHolder.getFinalQortFromQora(); holderReward -= adjustment; newQortFromQoraBalance -= adjustment; - block.repository.getAccountRepository().deleteQortFromQoraInfo(qoraHolder.getAddress()); + block.repository.getAccountRepository().deleteQortFromQoraInfo(qoraHolderAddress); long finalAdjustedHolderReward = holderReward; LOGGER.trace(() -> String.format("QORA holder %s final share %s was at height %d", - qoraHolder.getAddress(), Amounts.prettyAmount(finalAdjustedHolderReward), block.blockData.getHeight())); + qoraHolderAddress, Amounts.prettyAmount(finalAdjustedHolderReward), block.blockData.getHeight())); } } - balanceChanges.merge(qoraHolderAccount, holderReward, Long::sum); + balanceChanges.merge(qoraHolderAddress, holderReward, Long::sum); - if (newQortFromQoraBalance > 0) - qoraHolderAccount.setConfirmedBalance(Asset.QORT_FROM_QORA, newQortFromQoraBalance); - else - // Remove QORT_FROM_QORA balance as it's zero - qoraHolderAccount.deleteBalance(Asset.QORT_FROM_QORA); + // Add to batched QORT_FROM_QORA balance update list + newQortFromQoraBalances.add(new AccountBalanceData(qoraHolderAddress, Asset.QORT_FROM_QORA, newQortFromQoraBalance)); sharedAmount += holderReward; } + // Perform batched update of QORT_FROM_QORA balances + block.repository.getAccountRepository().setAssetBalances(newQortFromQoraBalances); + return sharedAmount; } @@ -1909,4 +2029,38 @@ private void postBlockTidy() throws DataException { this.repository.getAccountRepository().tidy(); } + private void logDebugInfo() { + try { + // Avoid calculations if possible. We have to check against INFO here, since Level.isMoreSpecificThan() confusingly uses <= rather than just < + if (LOGGER.getLevel().isMoreSpecificThan(Level.INFO)) + return; + + if (this.repository == null || this.getMinter() == null || this.getBlockData() == null) + return; + + int minterLevel = Account.getRewardShareEffectiveMintingLevel(this.repository, this.getMinter().getPublicKey()); + + LOGGER.debug(String.format("======= BLOCK %d (%.8s) =======", this.getBlockData().getHeight(), Base58.encode(this.getSignature()))); + LOGGER.debug(String.format("Timestamp: %d", this.getBlockData().getTimestamp())); + LOGGER.debug(String.format("Minter level: %d", minterLevel)); + LOGGER.debug(String.format("Online accounts: %d", this.getBlockData().getOnlineAccountsCount())); + LOGGER.debug(String.format("AT count: %d", this.getBlockData().getATCount())); + + BlockSummaryData blockSummaryData = new BlockSummaryData(this.getBlockData()); + if (this.getParent() == null || this.getParent().getSignature() == null || blockSummaryData == null || minterLevel == 0) + return; + + blockSummaryData.setMinterLevel(minterLevel); + BigInteger blockWeight = calcBlockWeight(this.getParent().getHeight(), this.getParent().getSignature(), blockSummaryData); + BigInteger keyDistance = calcKeyDistance(this.getParent().getHeight(), this.getParent().getSignature(), blockSummaryData.getMinterPublicKey(), blockSummaryData.getMinterLevel()); + NumberFormat formatter = new DecimalFormat("0.###E0"); + + LOGGER.debug(String.format("Key distance: %s", formatter.format(keyDistance))); + LOGGER.debug(String.format("Weight: %s", formatter.format(blockWeight))); + + } catch (DataException e) { + LOGGER.info(() -> String.format("Unable to log block debugging info: %s", e.getMessage())); + } + } + } diff --git a/src/main/java/org/qortal/block/Block212937.java b/src/main/java/org/qortal/block/Block212937.java new file mode 100644 index 000000000..a53c9d313 --- /dev/null +++ b/src/main/java/org/qortal/block/Block212937.java @@ -0,0 +1,153 @@ +package org.qortal.block; + +import java.io.InputStream; +import java.util.List; +import java.util.stream.Collectors; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.UnmarshalException; +import javax.xml.bind.Unmarshaller; +import javax.xml.transform.stream.StreamSource; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.eclipse.persistence.jaxb.JAXBContextFactory; +import org.eclipse.persistence.jaxb.UnmarshallerProperties; +import org.qortal.data.account.AccountBalanceData; +import org.qortal.repository.DataException; + +/** + * Block 212937 + *

+ * Somehow a node minted a version of block 212937 that contained one transaction: + * a PAYMENT transaction that attempted to spend more QORT than that account had as QORT balance. + *

+ * This invalid transaction made block 212937 (rightly) invalid to several nodes, + * which refused to use that block. + * However, it seems there were no other nodes minting an alternative, valid block at that time + * and so the chain stalled for several nodes in the network. + *

+ * Additionally, the invalid block 212937 affected all new installations, regardless of whether + * they synchronized from scratch (block 1) or used an 'official release' bootstrap. + *

+ * After lengthy diagnosis, it was discovered that + * the invalid transaction seemed to rely on incorrect balances in a corrupted database. + * Copies of DB files containing the broken chain were also shared around, exacerbating the problem. + *

+ * There were three options: + *

    + *
  1. roll back the chain to last known valid block 212936 and re-mint empty blocks to current height
    2. + *
  2. keep existing chain, but apply database edits at block 212937 to allow current chain to be valid
    3. + *
  3. attempt to mint an alternative chain, retaining as many valid transactions as possible
    4. + *
+ *

+ * Option 1 was highly undesirable due to knock-on effects from wiping 700+ transactions, some of which + * might have affect cross-chain trades, although there were no cross-chain trade completed during + * the decision period. + *

+ * Option 3 was essentially a slightly better version of option 1 and rejected for similar reasons. + * Attempts at option 3 also rapidly hit cumulative problems with every replacement block due to + * differing block timestamps making some transactions, and then even some blocks themselves, invalid. + *

+ * This class is the implementation of option 2. + *

+ * The change in account balances are relatively small, see block-212937-deltas.json resource + * for actual values. These values were obtained by exporting the AccountBalances table from + * both versions of the database with chain at block 212936, and then comparing. The values were also + * tested by syncing both databases up to block 225500, re-exporting and re-comparing. + *

+ * The invalid block 212937 signature is: 2J3GVJjv...qavh6KkQ. + *

+ * The invalid transaction in block 212937 is: + *

+ * 

+   {
+      "amount" : "0.10788294",
+      "approvalStatus" : "NOT_REQUIRED",
+      "blockHeight" : 212937,
+      "creatorAddress" : "QLdw5uabviLJgRGkRiydAFmAtZzxHfNXSs",
+      "fee" : "0.00100000",
+      "recipient" : "QZi1mNHDbiLvsytxTgxDr9nhJe4pNZaWpw",
+      "reference" : "J6JukdTVuXZ3JYbHatfZzwxG2vSiZwVCPDzW5K7PsVQKRj8XZeDtqnkGCGGjaSQZ9bQMtV44ky88NnGM4YBQKU6",
+      "senderPublicKey" : "DBFfbD2M3uh4jPE5PaUcZVvNPfrrJzVB7seeEtBn5SPs",
+      "signature" : "qkitxdCEEnKt8w6wRfFixtErbXsxWE6zG2ESNhpqBdScikV1WxeA6WZTTMJVV4tCeZdBFXw3V1X5NVztv6LirWK",
+      "timestamp" : 1607863074904,
+      "txGroupId" : 0,
+      "type" : "PAYMENT"
+   }
+
+ *

+ * Account QLdw5uabviLJgRGkRiydAFmAtZzxHfNXSs attempted to spend 0.10888294 (including fees) + * when their QORT balance was really only 0.10886665. + *

+ * However, on the broken DB nodes, their balance + * seemed to be 0.10890293 which was sufficient to make the transaction valid. + */ +public final class Block212937 { + + private static final Logger LOGGER = LogManager.getLogger(Block212937.class); + private static final String ACCOUNT_DELTAS_SOURCE = "block-212937-deltas.json"; + + private static final List accountDeltas = readAccountDeltas(); + + private Block212937() { + /* Do not instantiate */ + } + + @SuppressWarnings("unchecked") + private static List readAccountDeltas() { + Unmarshaller unmarshaller; + + try { + // Create JAXB context aware of classes we need to unmarshal + JAXBContext jc = JAXBContextFactory.createContext(new Class[] { + AccountBalanceData.class + }, null); + + // Create unmarshaller + unmarshaller = jc.createUnmarshaller(); + + // Set the unmarshaller media type to JSON + unmarshaller.setProperty(UnmarshallerProperties.MEDIA_TYPE, "application/json"); + + // Tell unmarshaller that there's no JSON root element in the JSON input + unmarshaller.setProperty(UnmarshallerProperties.JSON_INCLUDE_ROOT, false); + } catch (JAXBException e) { + String message = "Failed to setup unmarshaller to read block 212937 deltas"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); + } + + ClassLoader classLoader = BlockChain.class.getClassLoader(); + InputStream in = classLoader.getResourceAsStream(ACCOUNT_DELTAS_SOURCE); + StreamSource jsonSource = new StreamSource(in); + + try { + // Attempt to unmarshal JSON stream to BlockChain config + return (List) unmarshaller.unmarshal(jsonSource, AccountBalanceData.class).getValue(); + } catch (UnmarshalException e) { + String message = "Failed to parse block 212937 deltas"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); + } catch (JAXBException e) { + String message = "Unexpected JAXB issue while processing block 212937 deltas"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); + } + } + + public static void processFix(Block block) throws DataException { + block.repository.getAccountRepository().modifyAssetBalances(accountDeltas); + } + + public static void orphanFix(Block block) throws DataException { + // Create inverse deltas + List inverseDeltas = accountDeltas.stream() + .map(delta -> new AccountBalanceData(delta.getAddress(), delta.getAssetId(), 0 - delta.getBalance())) + .collect(Collectors.toList()); + + block.repository.getAccountRepository().modifyAssetBalances(inverseDeltas); + } + +} diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 531d3ad4e..69779d963 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -4,10 +4,7 @@ import java.io.FileNotFoundException; import java.io.InputStream; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.locks.ReentrantLock; import javax.xml.bind.JAXBContext; @@ -27,12 +24,9 @@ import org.qortal.controller.Controller; import org.qortal.data.block.BlockData; import org.qortal.network.Network; -import org.qortal.repository.BlockRepository; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; +import org.qortal.repository.*; import org.qortal.settings.Settings; -import org.qortal.utils.NTP; +import org.qortal.utils.Base58; import org.qortal.utils.StringLongMapXmlAdapter; /** @@ -71,8 +65,18 @@ public class BlockChain { private GenesisBlock.GenesisInfo genesisInfo; public enum FeatureTrigger { + atFindNextTransactionFix, + newBlockSigHeight, + shareBinFix, + calcChainWeightTimestamp, + transactionV5Timestamp; } + // Custom transaction fees + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long nameRegistrationUnitFee; + private long nameRegistrationUnitFeeTimestamp; + /** Map of which blockchain features are enabled when (height/timestamp) */ @XmlJavaTypeAdapter(StringLongMapXmlAdapter.class) private Map featureTriggers; @@ -143,7 +147,8 @@ public static class BlockTimingByHeight { } private List blockTimingsByHeight; - private int minAccountLevelToMint = 1; + private int minAccountLevelToMint; + private int minAccountLevelForBlockSubmissions; private int minAccountLevelToRewardShare; private int maxRewardSharesPerMintingAccount; private int founderEffectiveMintingLevel; @@ -301,6 +306,16 @@ public int getMaxBlockSize() { return this.maxBlockSize; } + // Custom transaction fees + public long getNameRegistrationUnitFee() { + return this.nameRegistrationUnitFee; + } + + public long getNameRegistrationUnitFeeTimestamp() { + // FUTURE: we could use a separate structure to indicate fee adjustments for different transaction types + return this.nameRegistrationUnitFeeTimestamp; + } + /** Returns true if approval-needing transaction types require a txGroupId other than NO_GROUP. */ public boolean getRequireGroupForApproval() { return this.requireGroupForApproval; @@ -346,6 +361,10 @@ public int getMinAccountLevelToMint() { return this.minAccountLevelToMint; } + public int getMinAccountLevelForBlockSubmissions() { + return this.minAccountLevelForBlockSubmissions; + } + public int getMinAccountLevelToRewardShare() { return this.minAccountLevelToRewardShare; } @@ -372,6 +391,26 @@ public CiyamAtSettings getCiyamAtSettings() { // Convenience methods for specific blockchain feature triggers + public int getAtFindNextTransactionFixHeight() { + return this.featureTriggers.get(FeatureTrigger.atFindNextTransactionFix.name()).intValue(); + } + + public int getNewBlockSigHeight() { + return this.featureTriggers.get(FeatureTrigger.newBlockSigHeight.name()).intValue(); + } + + public int getShareBinFixHeight() { + return this.featureTriggers.get(FeatureTrigger.shareBinFix.name()).intValue(); + } + + public long getCalcChainWeightTimestamp() { + return this.featureTriggers.get(FeatureTrigger.calcChainWeightTimestamp.name()).longValue(); + } + + public long getTransactionV5Timestamp() { + return this.featureTriggers.get(FeatureTrigger.transactionV5Timestamp.name()).longValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { @@ -482,30 +521,110 @@ private void fixUp() { } /** - * Some sort start-up/initialization/checking method. + * Some sort of start-up/initialization/checking method. * * @throws SQLException */ public static void validate() throws DataException { - // Check first block is Genesis Block - if (!isGenesisBlockValid()) - rebuildBlockchain(); + + boolean isTopOnly = Settings.getInstance().isTopOnly(); + boolean archiveEnabled = Settings.getInstance().isArchiveEnabled(); + boolean canBootstrap = Settings.getInstance().getBootstrap(); + boolean needsArchiveRebuild = false; + BlockData chainTip; try (final Repository repository = RepositoryManager.getRepository()) { - BlockData detachedBlockData = repository.getBlockRepository().getDetachedBlockSignature(); + chainTip = repository.getBlockRepository().getLastBlock(); + + // Ensure archive is (at least partially) intact, and force a bootstrap if it isn't + if (!isTopOnly && archiveEnabled && canBootstrap) { + needsArchiveRebuild = (repository.getBlockArchiveRepository().fromHeight(2) == null); + if (needsArchiveRebuild) { + LOGGER.info("Couldn't retrieve block 2 from archive. Bootstrapping..."); + + // If there are minting accounts, make sure to back them up + // Don't backup if there are no minting accounts, as this can cause problems + if (!repository.getAccountRepository().getMintingAccounts().isEmpty()) { + Controller.getInstance().exportRepositoryData(); + } + } + } + } - if (detachedBlockData != null) { - LOGGER.error(String.format("Block %d's reference does not match any block's signature", detachedBlockData.getHeight())); + boolean hasBlocks = (chainTip != null && chainTip.getHeight() > 1); - // Wait for blockchain lock (whereas orphan() only tries to get lock) - ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - blockchainLock.lock(); + if (isTopOnly && hasBlocks) { + // Top-only mode is enabled and we have blocks, so it's possible that the genesis block has been pruned + // It's best not to validate it, and there's no real need to + } else { + // Check first block is Genesis Block + if (!isGenesisBlockValid() || needsArchiveRebuild) { try { - LOGGER.info(String.format("Orphaning back to block %d", detachedBlockData.getHeight() - 1)); - orphan(detachedBlockData.getHeight() - 1); - } finally { - blockchainLock.unlock(); + rebuildBlockchain(); + + } catch (InterruptedException e) { + throw new DataException(String.format("Interrupted when trying to rebuild blockchain: %s", e.getMessage())); + } + } + } + + // We need to create a new connection, as the previous repository and its connections may be been + // closed by rebuildBlockchain() if a bootstrap was applied + try (final Repository repository = RepositoryManager.getRepository()) { + repository.checkConsistency(); + + // Set the number of blocks to validate based on the pruned state of the chain + // If pruned, subtract an extra 10 to allow room for error + int blocksToValidate = (isTopOnly || archiveEnabled) ? Settings.getInstance().getPruneBlockLimit() - 10 : 1440; + + int startHeight = Math.max(repository.getBlockRepository().getBlockchainHeight() - blocksToValidate, 1); + BlockData detachedBlockData = repository.getBlockRepository().getDetachedBlockSignature(startHeight); + + if (detachedBlockData != null) { + LOGGER.error(String.format("Block %d's reference does not match any block's signature", + detachedBlockData.getHeight())); + LOGGER.error(String.format("Your chain may be invalid and you should consider bootstrapping" + + " or re-syncing from genesis.")); + } + } + } + + /** + * More thorough blockchain validation method. Useful for validating bootstraps. + * A DataException is thrown if anything is invalid. + * + * @throws DataException + */ + public static void validateAllBlocks() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + BlockData chainTip = repository.getBlockRepository().getLastBlock(); + final int chainTipHeight = chainTip.getHeight(); + final int oldestBlock = 1; // TODO: increase if in pruning mode + byte[] lastReference = null; + + for (int height = chainTipHeight; height > oldestBlock; height--) { + BlockData blockData = repository.getBlockRepository().fromHeight(height); + if (blockData == null) { + blockData = repository.getBlockArchiveRepository().fromHeight(height); + } + + if (blockData == null) { + String error = String.format("Missing block at height %d", height); + LOGGER.error(error); + throw new DataException(error); + } + + if (height != chainTipHeight) { + // Check reference + if (!Arrays.equals(blockData.getSignature(), lastReference)) { + String error = String.format("Invalid reference for block at height %d: %s (should be %s)", + height, Base58.encode(blockData.getReference()), Base58.encode(lastReference)); + LOGGER.error(error); + throw new DataException(error); + } } + + lastReference = blockData.getReference(); } } } @@ -528,9 +647,18 @@ private static boolean isGenesisBlockValid() { } } - private static void rebuildBlockchain() throws DataException { + private static void rebuildBlockchain() throws DataException, InterruptedException { + boolean shouldBootstrap = Settings.getInstance().getBootstrap(); + if (shouldBootstrap) { + // Settings indicate that we should apply a bootstrap rather than rebuilding and syncing from genesis + Bootstrap bootstrap = new Bootstrap(); + bootstrap.startImport(); + return; + } + // (Re)build repository - RepositoryManager.rebuild(); + if (!RepositoryManager.wasPristineAtOpen()) + RepositoryManager.rebuild(); try (final Repository repository = RepositoryManager.getRepository()) { GenesisBlock genesisBlock = GenesisBlock.getInstance(repository); @@ -552,48 +680,25 @@ public static boolean orphan(int targetHeight) throws DataException { try { try (final Repository repository = RepositoryManager.getRepository()) { - for (int height = repository.getBlockRepository().getBlockchainHeight(); height > targetHeight; --height) { + int height = repository.getBlockRepository().getBlockchainHeight(); + BlockData orphanBlockData = repository.getBlockRepository().fromHeight(height); + + while (height > targetHeight) { LOGGER.info(String.format("Forcably orphaning block %d", height)); - BlockData blockData = repository.getBlockRepository().fromHeight(height); - Block block = new Block(repository, blockData); + Block block = new Block(repository, orphanBlockData); block.orphan(); - repository.saveChanges(); - } - - BlockData lastBlockData = repository.getBlockRepository().getLastBlock(); - Controller.getInstance().setChainTip(lastBlockData); - return true; - } - } finally { - blockchainLock.unlock(); - } - } - - public static void trimOldOnlineAccountsSignatures() { - final Long now = NTP.getTime(); - if (now == null) - return; - - ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - if (!blockchainLock.tryLock()) - // Too busy to trim right now, try again later - return; - - try { - try (final Repository repository = RepositoryManager.tryRepository()) { - if (repository == null) - return; + repository.saveChanges(); - int numBlocksTrimmed = repository.getBlockRepository().trimOldOnlineAccountsSignatures(now - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime()); + --height; + orphanBlockData = repository.getBlockRepository().fromHeight(height); - if (numBlocksTrimmed > 0) - LOGGER.debug(String.format("Trimmed old online accounts signatures from %d block%s", numBlocksTrimmed, (numBlocksTrimmed != 1 ? "s" : ""))); + repository.discardChanges(); // clear transaction status to prevent deadlocks + Controller.getInstance().onOrphanedBlock(orphanBlockData); + } - repository.saveChanges(); - } catch (DataException e) { - LOGGER.warn(String.format("Repository issue trying to trim old online accounts signatures: %s", e.getMessage())); + return true; } } finally { blockchainLock.unlock(); diff --git a/src/main/java/org/qortal/block/InvalidNameRegistrationBlocks.java b/src/main/java/org/qortal/block/InvalidNameRegistrationBlocks.java new file mode 100644 index 000000000..ebef366fe --- /dev/null +++ b/src/main/java/org/qortal/block/InvalidNameRegistrationBlocks.java @@ -0,0 +1,114 @@ +package org.qortal.block; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.naming.Name; +import org.qortal.repository.DataException; + +import java.util.HashMap; +import java.util.Map; + +/** + * Invalid Name Registration Blocks + *

+ * A node minted a version of block 535658 that contained one transaction: + * a REGISTER_NAME transaction that attempted to register a name that was already registered. + *

+ * This invalid transaction made block 535658 (rightly) invalid to several nodes, + * which refused to use that block. + * However, it seems there were no other nodes minting an alternative, valid block at that time + * and so the chain stalled for several nodes in the network. + *

+ * Additionally, the invalid block 535658 affected all new installations, regardless of whether + * they synchronized from scratch (block 1) or used an 'official release' bootstrap. + *

+ * The diagnosis found the following: + * - The original problem occurred in block 535205 where for some unknown reason many nodes didn't + * add the name from a REGISTER_NAME transaction to their Names table. + * - As a result, those nodes had a corrupt db, because they weren't holding a record of the name. + * - This invalid db then caused them to treat a candidate for block 535658 as valid when it + * should have been invalid. + * - As such, the chain continued on with a technically invalid block in it, for a subset of the network + *

+ * As with block 212937, there were three options, but the only feasible one was to apply edits to block + * 535658 to make it valid. There were several cross-chain trades completed after this block, so doing + * any kind of rollback was out of the question. + *

+ * To complicate things further, a custom data field was used for the first REGISTER_NAME transaction, + * and the default data field was used for the second. So it was important that all nodes ended up with + * the exact same data regardless of how they arrived there. + *

+ * The invalid block 535658 signature is: 3oiuDhok...NdXvCLEV. + *

+ * The invalid transaction in block 212937 is: + *

+ * 

+	 {
+		 "type": "REGISTER_NAME",
+		 "timestamp": 1630739437517,
+		 "reference": "4peRechwSPxP6UkRj9Y8ox9YxkWb34sWk5zyMc1WyMxEsACxD4Gmm7LZVsQ6Skpze8QCSBMZasvEZg6RgdqkyADW",
+		 "fee": "0.00100000",
+		 "signature": "2t1CryCog8KPDBarzY5fDCKu499nfnUcGrz4Lz4w5wNb5nWqm7y126P48dChYY7huhufcBV3RJPkgKP4Ywxc1gXx",
+		 "txGroupId": 0,
+		 "blockHeight": 535658,
+		 "approvalStatus": "NOT_REQUIRED",
+		 "creatorAddress": "Qbx9ojxv7XNi1xDMWzzw7xDvd1zYW6SKFB",
+		 "registrantPublicKey": "HJqGEf6cW695Xun4ydhkB2excGFwsDxznhNCRHZStyyx",
+		 "name": "Qplay",
+		 "data": "Registered Name on the Qortal Chain"
+	 }
+
+ *

+ * Account Qbx9ojxv7XNi1xDMWzzw7xDvd1zYW6SKFB attempted to register the name Qplay + * when they had already registered it 12 hours before in block 535205. + *

+ * However, on the broken DB nodes, their Names table was missing a record for the `Qplay` name + * which was sufficient to make the transaction valid. + * + * This problem then occurred two more times, in blocks 536140 and 541334 + * To reduce duplication, I have combined all three block fixes into a single class + * + */ +public final class InvalidNameRegistrationBlocks { + + private static final Logger LOGGER = LogManager.getLogger(InvalidNameRegistrationBlocks.class); + + public static Map invalidBlocksNamesMap = new HashMap() + { + { + put(535658, "Qplay"); + put(536140, "Qweb"); + put(541334, "Qithub"); + } + }; + + private InvalidNameRegistrationBlocks() { + /* Do not instantiate */ + } + + public static boolean isAffectedBlock(int height) { + return (invalidBlocksNamesMap.containsKey(height)); + } + + public static void processFix(Block block) throws DataException { + Integer blockHeight = block.getBlockData().getHeight(); + String invalidName = invalidBlocksNamesMap.get(blockHeight); + if (invalidName == null) { + throw new DataException(String.format("Unable to lookup invalid name for block height %d", blockHeight)); + } + + // Unregister the existing name record if it exists + // This ensures that the duplicate name is considered valid, and therefore + // the second (i.e. duplicate) REGISTER_NAME transaction data is applied. + // Both were issued by the same user account, so there is no conflict. + Name name = new Name(block.repository, invalidName); + name.unregister(); + + LOGGER.debug("Applied name registration patch for block {}", blockHeight); + } + + // Note: + // There is no need to write an orphanFix() method, as we do not have + // the necessary ATStatesData to orphan back this far anyway + +} diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java deleted file mode 100644 index 61447dbc8..000000000 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.qortal.controller; - -import java.util.Arrays; -import java.util.List; -import java.util.Random; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; -import org.qortal.data.transaction.ArbitraryTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.transaction.ArbitraryTransaction; -import org.qortal.transaction.Transaction.TransactionType; - -public class ArbitraryDataManager extends Thread { - - private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataManager.class); - private static final List ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY); - - private static ArbitraryDataManager instance; - - private volatile boolean isStopping = false; - - private ArbitraryDataManager() { - } - - public static ArbitraryDataManager getInstance() { - if (instance == null) - instance = new ArbitraryDataManager(); - - return instance; - } - - @Override - public void run() { - Thread.currentThread().setName("Arbitrary Data Manager"); - - try { - while (!isStopping) { - Thread.sleep(2000); - - // Any arbitrary transactions we want to fetch data for? - try (final Repository repository = RepositoryManager.getRepository()) { - List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, null, null, true); - if (signatures == null || signatures.isEmpty()) - continue; - - // Filter out those that already have local data - signatures.removeIf(signature -> hasLocalData(repository, signature)); - - if (signatures.isEmpty()) - continue; - - // Pick one at random - final int index = new Random().nextInt(signatures.size()); - byte[] signature = signatures.get(index); - - Controller.getInstance().fetchArbitraryData(signature); - } catch (DataException e) { - LOGGER.error("Repository issue when fetching arbitrary transaction data", e); - } - } - } catch (InterruptedException e) { - // Fall-through to exit thread... - } - } - - public void shutdown() { - isStopping = true; - this.interrupt(); - } - - private boolean hasLocalData(final Repository repository, final byte[] signature) { - try { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (!(transactionData instanceof ArbitraryTransactionData)) - return true; - - ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); - - return arbitraryTransaction.isDataLocal(); - } catch (DataException e) { - LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e); - return true; - } - } - -} diff --git a/src/main/java/org/qortal/controller/AutoUpdate.java b/src/main/java/org/qortal/controller/AutoUpdate.java index 61dcc46d9..f07e82d17 100644 --- a/src/main/java/org/qortal/controller/AutoUpdate.java +++ b/src/main/java/org/qortal/controller/AutoUpdate.java @@ -14,6 +14,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.TimeoutException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -214,8 +215,18 @@ private static boolean attemptUpdate(byte[] commitHash, byte[] downloadHash, Str return false; // failed - try another repo } - // Give repository a chance to backup in case things go badly wrong - RepositoryManager.backup(true); + // Give repository a chance to backup in case things go badly wrong (if enabled) + if (Settings.getInstance().getRepositoryBackupInterval() > 0) { + try { + // Timeout if the database isn't ready for backing up after 60 seconds + long timeout = 60 * 1000L; + RepositoryManager.backup(true, "backup", timeout); + + } catch (TimeoutException e) { + LOGGER.info("Attempt to backup repository failed due to timeout: {}", e.getMessage()); + // Continue with the auto update anyway... + } + } // Call ApplyUpdate to end this process (unlocking current JAR so it can be replaced) String javaHome = System.getProperty("java.home"); diff --git a/src/main/java/org/qortal/block/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java similarity index 58% rename from src/main/java/org/qortal/block/BlockMinter.java rename to src/main/java/org/qortal/controller/BlockMinter.java index 3542be5ec..d7d1dd487 100644 --- a/src/main/java/org/qortal/block/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -1,6 +1,8 @@ -package org.qortal.block; +package org.qortal.controller; import java.math.BigInteger; +import java.text.DecimalFormat; +import java.text.NumberFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -13,12 +15,14 @@ import org.apache.logging.log4j.Logger; import org.qortal.account.Account; import org.qortal.account.PrivateKeyAccount; +import org.qortal.block.Block; import org.qortal.block.Block.ValidationResult; -import org.qortal.controller.Controller; +import org.qortal.block.BlockChain; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; +import org.qortal.data.block.CommonBlockData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; import org.qortal.network.Peer; @@ -43,6 +47,9 @@ public class BlockMinter extends Thread { private static Long lastLogTimestamp; private static Long logTimeout; + // Recovery + public static final long INVALID_BLOCK_RECOVERY_TIMEOUT = 10 * 60 * 1000L; // ms + // Constructors public BlockMinter() { @@ -60,7 +67,7 @@ public void run() { List unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(); for (TransactionData transactionData : unconfirmedTransactions) { - LOGGER.trace(String.format("Deleting unconfirmed transaction %s", Base58.encode(transactionData.getSignature()))); + LOGGER.trace(() -> String.format("Deleting unconfirmed transaction %s", Base58.encode(transactionData.getSignature()))); repository.getTransactionRepository().delete(transactionData); } @@ -69,7 +76,11 @@ public void run() { // Going to need this a lot... BlockRepository blockRepository = repository.getBlockRepository(); - Block previousBlock = null; + BlockData previousBlockData = null; + + // Vars to keep track of blocks that were skipped due to chain weight + byte[] parentSignatureForLastLowWeightBlock = null; + Long timeOfLastLowWeightBlock = null; List newBlocks = new ArrayList<>(); @@ -115,7 +126,7 @@ public void run() { RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey()); if (rewardShareData == null) { - // Reward-share doesn't even exist - probably not a good sign + // Reward-share doesn't exist - probably cancelled but not yet removed from node's list of minting accounts madi.remove(); continue; } @@ -126,6 +137,15 @@ public void run() { madi.remove(); continue; } + + // Optional (non-validated) prevention of block submissions below a defined level. + // This is an unvalidated version of Blockchain.minAccountLevelToMint + // and exists only to reduce block candidates by default. + int level = mintingAccount.getEffectiveMintingLevel(); + if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) { + madi.remove(); + continue; + } } List peers = Network.getInstance().getHandshakedPeers(); @@ -134,40 +154,78 @@ public void run() { // Disregard peers that have "misbehaved" recently peers.removeIf(Controller.hasMisbehaved); - // Disregard peers that don't have a recent block - peers.removeIf(Controller.hasNoRecentBlock); + // Disregard peers that don't have a recent block, but only if we're not in recovery mode. + // In that mode, we want to allow minting on top of older blocks, to recover stalled networks. + if (Synchronizer.getInstance().getRecoveryMode() == false) + peers.removeIf(Controller.hasNoRecentBlock); // Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from? if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) continue; - // If our latest block isn't recent then we need to synchronize instead of minting. + // If we are stuck on an invalid block, we should allow an alternative to be minted + boolean recoverInvalidBlock = false; + if (Synchronizer.getInstance().timeInvalidBlockLastReceived != null) { + // We've had at least one invalid block + long timeSinceLastValidBlock = NTP.getTime() - Synchronizer.getInstance().timeValidBlockLastReceived; + long timeSinceLastInvalidBlock = NTP.getTime() - Synchronizer.getInstance().timeInvalidBlockLastReceived; + if (timeSinceLastValidBlock > INVALID_BLOCK_RECOVERY_TIMEOUT) { + if (timeSinceLastInvalidBlock < INVALID_BLOCK_RECOVERY_TIMEOUT) { + // Last valid block was more than 10 mins ago, but we've had an invalid block since then + // Assume that the chain has stalled because there is no alternative valid candidate + // Enter recovery mode to allow alternative, valid candidates to be minted + recoverInvalidBlock = true; + } + } + } + + // If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode. if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp) - continue; + if (Synchronizer.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false) + continue; // There are enough peers with a recent block and our latest block is recent // so go ahead and mint a block if possible. isMintingPossible = true; // Check blockchain hasn't changed - if (previousBlock == null || !Arrays.equals(previousBlock.getSignature(), lastBlockData.getSignature())) { - previousBlock = new Block(repository, lastBlockData); + if (previousBlockData == null || !Arrays.equals(previousBlockData.getSignature(), lastBlockData.getSignature())) { + previousBlockData = lastBlockData; newBlocks.clear(); // Reduce log timeout logTimeout = 10 * 1000L; + + // Last low weight block is no longer valid + parentSignatureForLastLowWeightBlock = null; } // Discard accounts we have already built blocks with mintingAccountsData.removeIf(mintingAccountData -> newBlocks.stream().anyMatch(newBlock -> Arrays.equals(newBlock.getBlockData().getMinterPublicKey(), mintingAccountData.getPublicKey()))); // Do we need to build any potential new blocks? - List mintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList()); + List newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList()); - for (PrivateKeyAccount mintingAccount : mintingAccounts) { + // We might need to sit the next block out, if one of our minting accounts signed the previous one + final byte[] previousBlockMinter = previousBlockData.getMinterPublicKey(); + final boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter)); + if (mintedLastBlock) { + LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one")); + continue; + } + + if (parentSignatureForLastLowWeightBlock != null) { + // The last iteration found a higher weight block in the network, so sleep for a while + // to allow is to sync the higher weight chain. We are sleeping here rather than when + // detected as we don't want to hold the blockchain lock open. + LOGGER.info("Sleeping for 10 seconds..."); + Thread.sleep(10 * 1000L); + } + + for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) { // First block does the AT heavy-lifting if (newBlocks.isEmpty()) { - Block newBlock = Block.mint(repository, previousBlock.getBlockData(), mintingAccount); + Block newBlock = Block.mint(repository, previousBlockData, mintingAccount); if (newBlock == null) { // For some reason we can't mint right now moderatedLog(() -> LOGGER.error("Couldn't build a to-be-minted block")); @@ -195,7 +253,7 @@ public void run() { // Make sure we're the only thread modifying the blockchain ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); if (!blockchainLock.tryLock(30, TimeUnit.SECONDS)) { - LOGGER.warn("Couldn't acquire blockchain lock even after waiting 30 seconds"); + LOGGER.debug("Couldn't acquire blockchain lock even after waiting 30 seconds"); continue; } @@ -218,6 +276,8 @@ public void run() { if (testBlock.isTimestampValid() != ValidationResult.OK) continue; + testBlock.preProcess(); + // Is new block valid yet? (Before adding unconfirmed transactions) ValidationResult result = testBlock.isValid(); if (result != ValidationResult.OK) { @@ -233,8 +293,8 @@ public void run() { continue; // Pick best block - final int parentHeight = previousBlock.getBlockData().getHeight(); - final byte[] parentBlockSignature = previousBlock.getSignature(); + final int parentHeight = previousBlockData.getHeight(); + final byte[] parentBlockSignature = previousBlockData.getSignature(); BigInteger bestWeight = null; @@ -253,6 +313,44 @@ public void run() { } } + try { + if (this.higherWeightChainExists(repository, bestWeight)) { + + // Check if the base block has updated since the last time we were here + if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null || + !Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) { + // We've switched to a different chain, so reset the timer + timeOfLastLowWeightBlock = NTP.getTime(); + } + parentSignatureForLastLowWeightBlock = previousBlockData.getSignature(); + + // If less than 30 seconds has passed since first detection the higher weight chain, + // we should skip our block submission to give us the opportunity to sync to the better chain + if (NTP.getTime() - timeOfLastLowWeightBlock < 30*1000L) { + LOGGER.info("Higher weight chain found in peers, so not signing a block this round"); + LOGGER.info("Time since detected: {}", NTP.getTime() - timeOfLastLowWeightBlock); + continue; + } + else { + // More than 30 seconds have passed, so we should submit our block candidate anyway. + LOGGER.info("More than 30 seconds passed, so proceeding to submit block candidate..."); + } + } + else { + LOGGER.debug("No higher weight chain found in peers"); + } + } catch (DataException e) { + LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway..."); + } + + // Discard any uncommitted changes as a result of the higher weight chain detection + repository.discardChanges(); + + // Clear variables that track low weight blocks + parentSignatureForLastLowWeightBlock = null; + timeOfLastLowWeightBlock = null; + + // Add unconfirmed transactions addUnconfirmedTransactions(repository, newBlock); @@ -274,28 +372,33 @@ public void run() { try { newBlock.process(); - LOGGER.info(String.format("Minted new block: %d", newBlock.getBlockData().getHeight())); repository.saveChanges(); + LOGGER.info(String.format("Minted new block: %d", newBlock.getBlockData().getHeight())); + RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey()); if (rewardShareData != null) { - LOGGER.info(String.format("Minted block %d, sig %.8s by %s on behalf of %s", + LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s", newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature()), + Base58.encode(newBlock.getParent().getSignature()), rewardShareData.getMinter(), rewardShareData.getRecipient())); } else { - LOGGER.info(String.format("Minted block %d, sig %.8s by %s", + LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s", newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature()), + Base58.encode(newBlock.getParent().getSignature()), newBlock.getMinter().getAddress())); } - repository.saveChanges(); - - // Notify controller + // Notify network after we're released blockchain lock newBlockMinted = true; + + // Notify Controller + repository.discardChanges(); // clear transaction status to prevent deadlocks + Controller.getInstance().onNewBlock(newBlock.getBlockData()); } catch (DataException e) { // Unable to process block - report and discard LOGGER.error("Unable to process newly minted block?", e); @@ -305,8 +408,13 @@ public void run() { blockchainLock.unlock(); } - if (newBlockMinted) - Controller.getInstance().onNewBlock(newBlock.getBlockData()); + if (newBlockMinted) { + // Broadcast our new chain to network + BlockData newBlockData = newBlock.getBlockData(); + + Network network = Network.getInstance(); + network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newBlockData)); + } } } catch (DataException e) { LOGGER.warn("Repository issue while running block minter", e); @@ -399,7 +507,8 @@ public static Block mintTestingBlockRetainingTimestamps(Repository repository, P // Add to blockchain newBlock.process(); - LOGGER.info(String.format("Minted new test block: %d", newBlock.getBlockData().getHeight())); + LOGGER.info(String.format("Minted new test block: %d sig: %.8s", + newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature()))); repository.saveChanges(); @@ -409,6 +518,61 @@ public static Block mintTestingBlockRetainingTimestamps(Repository repository, P } } + private BigInteger getOurChainWeightSinceBlock(Repository repository, BlockSummaryData commonBlock, List peerBlockSummaries) throws DataException { + final int commonBlockHeight = commonBlock.getHeight(); + final byte[] commonBlockSig = commonBlock.getSignature(); + int mutualHeight = commonBlockHeight; + + // Fetch our corresponding block summaries + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + List ourBlockSummaries = repository.getBlockRepository() + .getBlockSummaries(commonBlockHeight + 1, ourLatestBlockData.getHeight()); + if (!ourBlockSummaries.isEmpty()) { + Synchronizer.getInstance().populateBlockSummariesMinterLevels(repository, ourBlockSummaries); + } + + if (ourBlockSummaries != null && peerBlockSummaries != null) { + mutualHeight += Math.min(ourBlockSummaries.size(), peerBlockSummaries.size()); + } + return Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight); + } + + private boolean higherWeightChainExists(Repository repository, BigInteger blockCandidateWeight) throws DataException { + if (blockCandidateWeight == null) { + // Can't make decisions without knowing the block candidate weight + return false; + } + NumberFormat formatter = new DecimalFormat("0.###E0"); + + List peers = Network.getInstance().getHandshakedPeers(); + // Loop through handshaked peers and check for any new block candidates + for (Peer peer : peers) { + if (peer.getCommonBlockData() != null && peer.getCommonBlockData().getCommonBlockSummary() != null) { + // This peer has common block data + CommonBlockData commonBlockData = peer.getCommonBlockData(); + BlockSummaryData commonBlockSummaryData = commonBlockData.getCommonBlockSummary(); + if (commonBlockData.getChainWeight() != null) { + // The synchronizer has calculated this peer's chain weight + BigInteger ourChainWeightSinceCommonBlock = this.getOurChainWeightSinceBlock(repository, commonBlockSummaryData, commonBlockData.getBlockSummariesAfterCommonBlock()); + BigInteger ourChainWeight = ourChainWeightSinceCommonBlock.add(blockCandidateWeight); + BigInteger peerChainWeight = commonBlockData.getChainWeight(); + if (peerChainWeight.compareTo(ourChainWeight) >= 0) { + // This peer has a higher weight chain than ours + LOGGER.debug("Peer {} is on a higher weight chain ({}) than ours ({})", peer, formatter.format(peerChainWeight), formatter.format(ourChainWeight)); + return true; + + } else { + LOGGER.debug("Peer {} is on a lower weight chain ({}) than ours ({})", peer, formatter.format(peerChainWeight), formatter.format(ourChainWeight)); + } + } else { + LOGGER.debug("Peer {} has no chain weight", peer); + } + } else { + LOGGER.debug("Peer {} has no common block data", peer); + } + } + return false; + } private static void moderatedLog(Runnable logFunction) { // We only log if logging at TRACE or previous log timeout has expired if (!LOGGER.isTraceEnabled() && lastLogTimestamp != null && lastLogTimestamp + logTimeout > System.currentTimeMillis()) diff --git a/src/main/java/org/qortal/controller/BlockNotifier.java b/src/main/java/org/qortal/controller/BlockNotifier.java deleted file mode 100644 index d4278d05a..000000000 --- a/src/main/java/org/qortal/controller/BlockNotifier.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.qortal.controller; - -import java.util.HashMap; -import java.util.Map; - -import org.eclipse.jetty.websocket.api.Session; -import org.qortal.data.block.BlockData; - -public class BlockNotifier { - - private static BlockNotifier instance; - - @FunctionalInterface - public interface Listener { - void notify(BlockData blockData); - } - - private Map listenersBySession = new HashMap<>(); - - private BlockNotifier() { - } - - public static synchronized BlockNotifier getInstance() { - if (instance == null) - instance = new BlockNotifier(); - - return instance; - } - - public synchronized void register(Session session, Listener listener) { - this.listenersBySession.put(session, listener); - } - - public synchronized void deregister(Session session) { - this.listenersBySession.remove(session); - } - - public synchronized void onNewBlock(BlockData blockData) { - for (Listener listener : this.listenersBySession.values()) - listener.notify(blockData); - } - -} diff --git a/src/main/java/org/qortal/controller/ChatNotifier.java b/src/main/java/org/qortal/controller/ChatNotifier.java index 57ca7cbfc..61146faa3 100644 --- a/src/main/java/org/qortal/controller/ChatNotifier.java +++ b/src/main/java/org/qortal/controller/ChatNotifier.java @@ -1,5 +1,7 @@ package org.qortal.controller; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -27,22 +29,34 @@ public static synchronized ChatNotifier getInstance() { return instance; } - public synchronized void register(Session session, Listener listener) { - this.listenersBySession.put(session, listener); + public void register(Session session, Listener listener) { + synchronized (this.listenersBySession) { + this.listenersBySession.put(session, listener); + } } - public synchronized void deregister(Session session) { - this.listenersBySession.remove(session); + public void deregister(Session session) { + synchronized (this.listenersBySession) { + this.listenersBySession.remove(session); + } } - public synchronized void onNewChatTransaction(ChatTransactionData chatTransactionData) { - for (Listener listener : this.listenersBySession.values()) + public void onNewChatTransaction(ChatTransactionData chatTransactionData) { + for (Listener listener : getAllListeners()) listener.notify(chatTransactionData); } - public synchronized void onGroupMembershipChange() { - for (Listener listener : this.listenersBySession.values()) + public void onGroupMembershipChange() { + for (Listener listener : getAllListeners()) listener.notify(null); } + private Collection getAllListeners() { + // Make a copy of listeners to both avoid concurrent modification + // and reduce synchronization time + synchronized (this.listenersBySession) { + return new ArrayList<>(this.listenersBySession.values()); + } + } + } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 6c9bac276..fb1cbd473 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -2,42 +2,47 @@ import java.awt.TrayIcon.MessageType; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.security.SecureRandom; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.Security; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Random; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import com.google.common.primitives.Longs; import org.qortal.account.Account; import org.qortal.account.PrivateKeyAccount; import org.qortal.account.PublicKeyAccount; import org.qortal.api.ApiService; +import org.qortal.api.DomainMapService; +import org.qortal.api.GatewayService; import org.qortal.block.Block; import org.qortal.block.BlockChain; -import org.qortal.block.BlockMinter; import org.qortal.block.BlockChain.BlockTimingByHeight; -import org.qortal.controller.Synchronizer.SynchronizationResult; -import org.qortal.crypto.Crypto; +import org.qortal.controller.arbitrary.*; +import org.qortal.controller.repository.PruneManager; +import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; +import org.qortal.controller.tradebot.TradeBot; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; import org.qortal.data.block.BlockData; @@ -45,53 +50,29 @@ import org.qortal.data.network.OnlineAccountData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; -import org.qortal.data.transaction.ArbitraryTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.data.transaction.ArbitraryTransactionData.DataType; import org.qortal.data.transaction.ChatTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; import org.qortal.globalization.Translator; import org.qortal.gui.Gui; import org.qortal.gui.SysTray; import org.qortal.network.Network; import org.qortal.network.Peer; -import org.qortal.network.message.ArbitraryDataMessage; -import org.qortal.network.message.BlockMessage; -import org.qortal.network.message.BlockSummariesMessage; -import org.qortal.network.message.GetArbitraryDataMessage; -import org.qortal.network.message.GetBlockMessage; -import org.qortal.network.message.GetBlockSummariesMessage; -import org.qortal.network.message.GetOnlineAccountsMessage; -import org.qortal.network.message.GetPeersMessage; -import org.qortal.network.message.GetSignaturesV2Message; -import org.qortal.network.message.GetTransactionMessage; -import org.qortal.network.message.GetUnconfirmedTransactionsMessage; -import org.qortal.network.message.HeightV2Message; -import org.qortal.network.message.Message; -import org.qortal.network.message.OnlineAccountsMessage; -import org.qortal.network.message.SignaturesMessage; -import org.qortal.network.message.TransactionMessage; -import org.qortal.network.message.TransactionSignaturesMessage; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryFactory; -import org.qortal.repository.RepositoryManager; +import org.qortal.network.message.*; +import org.qortal.repository.*; import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; -import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.utils.Base58; -import org.qortal.utils.ByteArray; -import org.qortal.utils.NTP; -import org.qortal.utils.Triple; - -import com.google.common.primitives.Longs; +import org.qortal.utils.*; public class Controller extends Thread { static { // This must go before any calls to LogManager/Logger + System.setProperty("log4j2.formatMsgNoLookups", "true"); System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); } @@ -100,24 +81,35 @@ public class Controller extends Thread { public static final String VERSION_PREFIX = "qortal-"; private static final Logger LOGGER = LogManager.getLogger(Controller.class); - private static final long MISBEHAVIOUR_COOLOFF = 10 * 60 * 1000L; // ms + public static final long MISBEHAVIOUR_COOLOFF = 10 * 60 * 1000L; // ms private static final int MAX_BLOCKCHAIN_TIP_AGE = 5; // blocks private static final Object shutdownLock = new Object(); private static final String repositoryUrlTemplate = "jdbc:hsqldb:file:%s" + File.separator + "blockchain;create=true;hsqldb.full_log_replay=true"; - private static final long ARBITRARY_REQUEST_TIMEOUT = 5 * 1000L; // ms private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms + private static final int MAX_INCOMING_TRANSACTIONS = 5000; + + /** Minimum time before considering an invalid unconfirmed transaction as "stale" */ + public static final long INVALID_TRANSACTION_STALE_TIMEOUT = 30 * 60 * 1000L; // ms + /** Minimum frequency to re-request stale unconfirmed transactions from peers, to recheck validity */ + public static final long INVALID_TRANSACTION_RECHECK_INTERVAL = 60 * 60 * 1000L; // ms\ + /** Minimum frequency to re-request expired unconfirmed transactions from peers, to recheck validity + * This mainly exists to stop expired transactions from bloating the list */ + public static final long EXPIRED_TRANSACTION_RECHECK_INTERVAL = 10 * 60 * 1000L; // ms // To do with online accounts list private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 1 * 60 * 1000L; // ms - private static final long ONLINE_TIMESTAMP_MODULUS = 5 * 60 * 1000L; + public static final long ONLINE_TIMESTAMP_MODULUS = 5 * 60 * 1000L; private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000L); + /** How many (latest) blocks' worth of online accounts we cache */ + private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 2; + private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L; + private static volatile boolean isStopping = false; private static BlockMinter blockMinter = null; - private static volatile boolean requestSync = false; private static volatile boolean requestSysTrayUpdate = true; private static Controller instance; @@ -128,48 +120,101 @@ public class Controller extends Thread { private ExecutorService callbackExecutor = Executors.newFixedThreadPool(3); private volatile boolean notifyGroupMembershipChange = false; - private volatile BlockData chainTip = null; + /** Latest blocks on our chain. Note: tail/last is the latest block. */ + private final Deque latestBlocks = new LinkedList<>(); + + /** Cache of BlockMessages, indexed by block signature */ + @SuppressWarnings("serial") + private final LinkedHashMap blockMessageCache = new LinkedHashMap<>() { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return this.size() > Settings.getInstance().getBlockCacheSize(); + } + }; private long repositoryBackupTimestamp = startTime; // ms + private long repositoryMaintenanceTimestamp = startTime; // ms + private long repositoryCheckpointTimestamp = startTime; // ms private long ntpCheckTimestamp = startTime; // ms private long deleteExpiredTimestamp = startTime + DELETE_EXPIRED_INTERVAL; // ms + private long onlineAccountsTasksTimestamp = startTime + ONLINE_ACCOUNTS_TASKS_INTERVAL; // ms /** Whether we can mint new blocks, as reported by BlockMinter. */ private volatile boolean isMintingPossible = false; - /** Whether we are attempting to synchronize. */ - private volatile boolean isSynchronizing = false; - /** Temporary estimate of synchronization progress for SysTray use. */ - private volatile int syncPercent = 0; + /** List of incoming transaction that are in the import queue */ + private List incomingTransactions = Collections.synchronizedList(new ArrayList<>()); - /** Latest block signatures from other peers that we know are on inferior chains. */ - List inferiorChainSignatures = new ArrayList<>(); - - /** - * Map of recent requests for ARBITRARY transaction data payloads. - *

- * Key is original request's message ID
- * Value is Triple<transaction signature in base58, first requesting peer, first request's timestamp> - *

- * If peer is null then either:
- *

    - *
  • we are the original requesting peer
    • - *
  • we have already sent data payload to original requesting peer.
    • - *
- * If signature is null then we have already received the data payload and either:
- *
    - *
  • we are the original requesting peer and have saved it locally
    • - *
  • we have forwarded the data payload (and maybe also saved it locally)
    • - *
- */ - private Map> arbitraryDataRequests = Collections.synchronizedMap(new HashMap<>()); + /** List of recent invalid unconfirmed transactions */ + private Map invalidUnconfirmedTransactions = Collections.synchronizedMap(new HashMap<>()); /** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly minted block. */ private final ReentrantLock blockchainLock = new ReentrantLock(); - /** Cache of 'online accounts' */ + /** Cache of current 'online accounts' */ List onlineAccounts = new ArrayList<>(); + /** Cache of latest blocks' online accounts */ + Deque> latestBlocksOnlineAccounts = new ArrayDeque<>(MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS); + + // Stats + @XmlAccessorType(XmlAccessType.FIELD) + public static class StatsSnapshot { + public static class GetBlockMessageStats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong cacheHits = new AtomicLong(); + public AtomicLong unknownBlocks = new AtomicLong(); + public AtomicLong cacheFills = new AtomicLong(); + + public GetBlockMessageStats() { + } + } + public GetBlockMessageStats getBlockMessageStats = new GetBlockMessageStats(); + + public static class GetBlockSummariesStats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong cacheHits = new AtomicLong(); + public AtomicLong fullyFromCache = new AtomicLong(); + + public GetBlockSummariesStats() { + } + } + public GetBlockSummariesStats getBlockSummariesStats = new GetBlockSummariesStats(); + + public static class GetBlockSignaturesV2Stats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong cacheHits = new AtomicLong(); + public AtomicLong fullyFromCache = new AtomicLong(); + + public GetBlockSignaturesV2Stats() { + } + } + public GetBlockSignaturesV2Stats getBlockSignaturesV2Stats = new GetBlockSignaturesV2Stats(); + + public static class GetArbitraryDataFileMessageStats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong unknownFiles = new AtomicLong(); + + public GetArbitraryDataFileMessageStats() { + } + } + public GetArbitraryDataFileMessageStats getArbitraryDataFileMessageStats = new GetArbitraryDataFileMessageStats(); + + public static class GetArbitraryDataFileListMessageStats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong unknownFiles = new AtomicLong(); + + public GetArbitraryDataFileListMessageStats() { + } + } + public GetArbitraryDataFileListMessageStats getArbitraryDataFileListMessageStats = new GetArbitraryDataFileListMessageStats(); + + public AtomicLong latestBlocksCacheRefills = new AtomicLong(); + + public StatsSnapshot() { + } + } + public final StatsSnapshot stats = new StatsSnapshot(); // Constructors @@ -181,17 +226,29 @@ private Controller(String[] args) { throw new RuntimeException("Can't read build.properties resource", e); } + // Determine build timestamp String buildTimestampProperty = properties.getProperty("build.timestamp"); - if (buildTimestampProperty == null) + if (buildTimestampProperty == null) { throw new RuntimeException("Can't read build.timestamp from build.properties resource"); - - this.buildTimestamp = LocalDateTime.parse(buildTimestampProperty, DateTimeFormatter.ofPattern("yyyyMMddHHmmss")).toEpochSecond(ZoneOffset.UTC); + } + if (buildTimestampProperty.startsWith("$")) { + // Maven vars haven't been replaced - this was most likely built using an IDE, not via mvn package + this.buildTimestamp = System.currentTimeMillis(); + buildTimestampProperty = "unknown"; + } else { + this.buildTimestamp = LocalDateTime.parse(buildTimestampProperty, DateTimeFormatter.ofPattern("yyyyMMddHHmmss")).toEpochSecond(ZoneOffset.UTC); + } LOGGER.info(String.format("Build timestamp: %s", buildTimestampProperty)); + // Determine build version String buildVersionProperty = properties.getProperty("build.version"); - if (buildVersionProperty == null) + if (buildVersionProperty == null) { throw new RuntimeException("Can't read build.version from build.properties resource"); - + } + if (buildVersionProperty.contains("${git.commit.id.abbrev}")) { + // Maven vars haven't been replaced - this was most likely built using an IDE, not via mvn package + buildVersionProperty = buildVersionProperty.replace("${git.commit.id.abbrev}", "debug"); + } this.buildVersion = VERSION_PREFIX + buildVersionProperty; LOGGER.info(String.format("Build version: %s", this.buildVersion)); @@ -224,23 +281,43 @@ public String getVersionString() { return this.buildVersion; } + public String getVersionStringWithoutPrefix() { + return this.buildVersion.replaceFirst(VERSION_PREFIX, ""); + } + /** Returns current blockchain height, or 0 if it's not available. */ public int getChainHeight() { - BlockData blockData = this.chainTip; - if (blockData == null) - return 0; + synchronized (this.latestBlocks) { + BlockData blockData = this.latestBlocks.peekLast(); + if (blockData == null) + return 0; - return blockData.getHeight(); + return blockData.getHeight(); + } } /** Returns highest block, or null if it's not available. */ public BlockData getChainTip() { - return this.chainTip; + synchronized (this.latestBlocks) { + return this.latestBlocks.peekLast(); + } } - /** Cache new blockchain tip. */ - public void setChainTip(BlockData blockData) { - this.chainTip = blockData; + public void refillLatestBlocksCache() throws DataException { + // Set initial chain height/tip + try (final Repository repository = RepositoryManager.getRepository()) { + BlockData blockData = repository.getBlockRepository().getLastBlock(); + int blockCacheSize = Settings.getInstance().getBlockCacheSize(); + + synchronized (this.latestBlocks) { + this.latestBlocks.clear(); + + for (int i = 0; i < blockCacheSize && blockData != null; ++i) { + this.latestBlocks.addFirst(blockData); + blockData = repository.getBlockRepository().fromHeight(blockData.getHeight() - 1); + } + } + } } public ReentrantLock getBlockchainLock() { @@ -251,7 +328,7 @@ public ReentrantLock getBlockchainLock() { return this.savedArgs; } - /* package */ static boolean isStopping() { + public static boolean isStopping() { return isStopping; } @@ -260,17 +337,11 @@ public boolean isMintingPossible() { return this.isMintingPossible; } - public boolean isSynchronizing() { - return this.isSynchronizing; - } - - public Integer getSyncPercent() { - return this.isSynchronizing ? this.syncPercent : null; - } - // Entry point public static void main(String[] args) { + LoggingUtils.fixLegacyLog4j2Properties(); + LOGGER.info("Starting up..."); // Potential GUI startup with splash screen, etc. @@ -303,6 +374,12 @@ public static void main(String[] args) { try { RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl()); RepositoryManager.setRepositoryFactory(repositoryFactory); + RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); + + try (final Repository repository = RepositoryManager.getRepository()) { + RepositoryManager.archive(repository); + RepositoryManager.prune(repository); + } } catch (DataException e) { // If exception has no cause then repository is in use by some other process. if (e.getCause() == null) { @@ -316,23 +393,31 @@ public static void main(String[] args) { return; // Not System.exit() so that GUI can display error } + // Rebuild Names table and check database integrity (if enabled) + NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck(); + namesDatabaseIntegrityCheck.rebuildAllNames(); + if (Settings.getInstance().isNamesIntegrityCheckEnabled()) { + namesDatabaseIntegrityCheck.runIntegrityCheck(); + } + LOGGER.info("Validating blockchain"); try { BlockChain.validate(); - // Set initial chain height/tip - try (final Repository repository = RepositoryManager.getRepository()) { - BlockData blockData = repository.getBlockRepository().getLastBlock(); - - Controller.getInstance().setChainTip(blockData); - LOGGER.info(String.format("Our chain height at start-up: %d", blockData.getHeight())); - } + Controller.getInstance().refillLatestBlocksCache(); + LOGGER.info(String.format("Our chain height at start-up: %d", Controller.getInstance().getChainHeight())); } catch (DataException e) { LOGGER.error("Couldn't validate blockchain", e); Gui.getInstance().fatalError("Blockchain validation issue", e); return; // Not System.exit() so that GUI can display error } + // Import current trade bot states and minting accounts if they exist + Controller.importRepositoryData(); + + // Add the initial peers to the repository if we don't have any + Controller.installInitialPeers(); + LOGGER.info("Starting controller"); Controller.getInstance().start(); @@ -356,13 +441,24 @@ public void run() { } }); + LOGGER.info("Starting synchronizer"); + Synchronizer.getInstance().start(); + LOGGER.info("Starting block minter"); blockMinter = new BlockMinter(); blockMinter.start(); - // Arbitrary transaction data manager - // LOGGER.info("Starting arbitrary-transaction data manager"); - // ArbitraryDataManager.getInstance().start(); + LOGGER.info("Starting trade-bot"); + TradeBot.getInstance(); + + // Arbitrary data controllers + LOGGER.info("Starting arbitrary-transaction controllers"); + ArbitraryDataManager.getInstance().start(); + ArbitraryDataFileManager.getInstance().start(); + ArbitraryDataBuildManager.getInstance().start(); + ArbitraryDataCleanupManager.getInstance().start(); + ArbitraryDataStorageManager.getInstance().start(); + ArbitraryDataRenderManager.getInstance().start(); // Auto-update service? if (Settings.getInstance().isAutoUpdateEnabled()) { @@ -381,6 +477,32 @@ public void run() { return; // Not System.exit() so that GUI can display error } + if (Settings.getInstance().isGatewayEnabled()) { + LOGGER.info(String.format("Starting gateway service on port %d", Settings.getInstance().getGatewayPort())); + try { + GatewayService gatewayService = GatewayService.getInstance(); + gatewayService.start(); + } catch (Exception e) { + LOGGER.error("Unable to start gateway service", e); + Controller.getInstance().shutdown(); + Gui.getInstance().fatalError("Gateway service failure", e); + return; // Not System.exit() so that GUI can display error + } + } + + if (Settings.getInstance().isDomainMapEnabled()) { + LOGGER.info(String.format("Starting domain map service on port %d", Settings.getInstance().getDomainMapPort())); + try { + DomainMapService domainMapService = DomainMapService.getInstance(); + domainMapService.start(); + } catch (Exception e) { + LOGGER.error("Unable to start domain map service", e); + Controller.getInstance().shutdown(); + Gui.getInstance().fatalError("Domain map service failure", e); + return; // Not System.exit() so that GUI can display error + } + } + // If GUI is enabled, we're no longer starting up but actually running now Gui.getInstance().notifyRunning(); } @@ -395,9 +517,14 @@ public static void secondaryMain(String[] args) { @Override public void run() { - Thread.currentThread().setName("Controller"); + Thread.currentThread().setName("Qortal"); final long repositoryBackupInterval = Settings.getInstance().getRepositoryBackupInterval(); + final long repositoryCheckpointInterval = Settings.getInstance().getRepositoryCheckpointInterval(); + long repositoryMaintenanceInterval = getRandomRepositoryMaintenanceInterval(); + + // Start executor service for trimming or pruning + PruneManager.getInstance().start(); try { while (!isStopping) { @@ -430,14 +557,22 @@ public void run() { } } - if (requestSync) { - requestSync = false; - potentiallySynchronize(); - } + // Process incoming transactions queue + processIncomingTransactionsQueue(); + // Clean up invalid incoming transactions list + cleanupInvalidTransactionsList(now); // Clean up arbitrary data request cache - final long requestMinimumTimestamp = now - ARBITRARY_REQUEST_TIMEOUT; - arbitraryDataRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); + ArbitraryDataManager.getInstance().cleanupRequestCache(now); + // Clean up arbitrary data queues and lists + ArbitraryDataBuildManager.getInstance().cleanupQueues(now); + + // Time to 'checkpoint' uncommitted repository writes? + if (now >= repositoryCheckpointTimestamp + repositoryCheckpointInterval) { + repositoryCheckpointTimestamp = now + repositoryCheckpointInterval; + + RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); + } // Give repository a chance to backup (if enabled) if (repositoryBackupInterval > 0 && now >= repositoryBackupTimestamp + repositoryBackupInterval) { @@ -448,7 +583,39 @@ public void run() { Translator.INSTANCE.translate("SysTray", "CREATING_BACKUP_OF_DB_FILES"), MessageType.INFO); - RepositoryManager.backup(true); + try { + // Timeout if the database isn't ready for backing up after 60 seconds + long timeout = 60 * 1000L; + RepositoryManager.backup(true, "backup", timeout); + + } catch (TimeoutException e) { + LOGGER.info("Attempt to backup repository failed due to timeout: {}", e.getMessage()); + } + } + + // Give repository a chance to perform maintenance (if enabled) + if (repositoryMaintenanceInterval > 0 && now >= repositoryMaintenanceTimestamp + repositoryMaintenanceInterval) { + repositoryMaintenanceTimestamp = now + repositoryMaintenanceInterval; + + if (Settings.getInstance().getShowMaintenanceNotification()) + SysTray.getInstance().showMessage(Translator.INSTANCE.translate("SysTray", "DB_MAINTENANCE"), + Translator.INSTANCE.translate("SysTray", "PERFORMING_DB_MAINTENANCE"), + MessageType.INFO); + + LOGGER.info("Starting scheduled repository maintenance. This can take a while..."); + try (final Repository repository = RepositoryManager.getRepository()) { + + // Timeout if the database isn't ready for maintenance after 60 seconds + long timeout = 60 * 1000L; + repository.performPeriodicMaintenance(timeout); + + LOGGER.info("Scheduled repository maintenance completed"); + } catch (DataException | TimeoutException e) { + LOGGER.error("Scheduled repository maintenance failed", e); + } + + // Get a new random interval + repositoryMaintenanceInterval = getRandomRepositoryMaintenanceInterval(); } // Prune stuck/slow/old peers @@ -471,7 +638,51 @@ public void run() { } } } catch (InterruptedException e) { + // Clear interrupted flag so we can shutdown trim threads + Thread.interrupted(); // Fall-through to exit + } finally { + PruneManager.getInstance().stop(); + } + } + + /** + * Import current trade bot states and minting accounts. + * This is needed because the user may have bootstrapped, or there could be a database inconsistency + * if the core crashed when computing the nonce during the start of the trade process. + */ + private static void importRepositoryData() { + try (final Repository repository = RepositoryManager.getRepository()) { + + String exportPath = Settings.getInstance().getExportPath(); + try { + Path importPath = Paths.get(exportPath, "TradeBotStates.json"); + repository.importDataFromFile(importPath.toString()); + } catch (FileNotFoundException e) { + // Do nothing, as the files will only exist in certain cases + } + + try { + Path importPath = Paths.get(exportPath, "MintingAccounts.json"); + repository.importDataFromFile(importPath.toString()); + } catch (FileNotFoundException e) { + // Do nothing, as the files will only exist in certain cases + } + repository.saveChanges(); + } + catch (DataException | IOException e) { + LOGGER.info("Unable to import data into repository: {}", e.getMessage()); + } + } + + private static void installInitialPeers() { + try (final Repository repository = RepositoryManager.getRepository()) { + if (repository.getNetworkRepository().getAllPeers().isEmpty()) { + Network.installInitialPeers(repository); + } + + } catch (DataException e) { + // Fail silently as this is an optional step } } @@ -499,132 +710,46 @@ public void run() { public static final Predicate hasInferiorChainTip = peer -> { final PeerChainTipData peerChainTipData = peer.getChainTipData(); - final List inferiorChainTips = getInstance().inferiorChainSignatures; + final List inferiorChainTips = Synchronizer.getInstance().inferiorChainSignatures; return peerChainTipData == null || peerChainTipData.getLastBlockSignature() == null || inferiorChainTips.contains(new ByteArray(peerChainTipData.getLastBlockSignature())); }; - private void potentiallySynchronize() throws InterruptedException { - List peers = Network.getInstance().getHandshakedPeers(); - - // Disregard peers that have "misbehaved" recently - peers.removeIf(hasMisbehaved); - - // Disregard peers that only have genesis block - peers.removeIf(hasOnlyGenesisBlock); - - // Disregard peers that don't have a recent block - peers.removeIf(hasNoRecentBlock); - - // Check we have enough peers to potentially synchronize - if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) - return; - - // Disregard peers that have no block signature or the same block signature as us - peers.removeIf(hasNoOrSameBlock); - - // Disregard peers that are on the same block as last sync attempt and we didn't like their chain - peers.removeIf(hasInferiorChainTip); - - if (peers.isEmpty()) - return; - - // Pick random peer to sync with - int index = new SecureRandom().nextInt(peers.size()); - Peer peer = peers.get(index); + public static final Predicate hasOldVersion = peer -> { + final String minPeerVersion = Settings.getInstance().getMinPeerVersion(); + return peer.isAtLeastVersion(minPeerVersion) == false; + }; - actuallySynchronize(peer, false); + private long getRandomRepositoryMaintenanceInterval() { + final long minInterval = Settings.getInstance().getRepositoryMaintenanceMinInterval(); + final long maxInterval = Settings.getInstance().getRepositoryMaintenanceMaxInterval(); + if (maxInterval == 0) { + return 0; + } + return (new Random().nextLong() % (maxInterval - minInterval)) + minInterval; } - public SynchronizationResult actuallySynchronize(Peer peer, boolean force) throws InterruptedException { - syncPercent = (this.chainTip.getHeight() * 100) / peer.getChainTipData().getLastHeight(); - isSynchronizing = true; - updateSysTray(); - - BlockData priorChainTip = this.chainTip; - - try { - SynchronizationResult syncResult = Synchronizer.getInstance().synchronize(peer, force); - switch (syncResult) { - case GENESIS_ONLY: - case NO_COMMON_BLOCK: - case TOO_DIVERGENT: - case INVALID_DATA: { - // These are more serious results that warrant a cool-off - LOGGER.info(String.format("Failed to synchronize with peer %s (%s) - cooling off", peer, syncResult.name())); - - // Don't use this peer again for a while - Network.getInstance().peerMisbehaved(peer); - break; - } - - case INFERIOR_CHAIN: { - // Update our list of inferior chain tips - ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature()); - if (!inferiorChainSignatures.contains(inferiorChainSignature)) - inferiorChainSignatures.add(inferiorChainSignature); - - // These are minor failure results so fine to try again - LOGGER.debug(() -> String.format("Refused to synchronize with peer %s (%s)", peer, syncResult.name())); - - // Notify peer of our superior chain - if (!peer.sendMessage(Network.getInstance().buildHeightMessage(peer, priorChainTip))) - peer.disconnect("failed to notify peer of our superior chain"); - break; - } - - case NO_REPLY: - case NO_BLOCKCHAIN_LOCK: - case REPOSITORY_ISSUE: - // These are minor failure results so fine to try again - LOGGER.debug(() -> String.format("Failed to synchronize with peer %s (%s)", peer, syncResult.name())); - break; - - case SHUTTING_DOWN: - // Just quietly exit - break; - - case OK: - requestSysTrayUpdate = true; - // fall-through... - case NOTHING_TO_DO: { - // Update our list of inferior chain tips - ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature()); - if (!inferiorChainSignatures.contains(inferiorChainSignature)) - inferiorChainSignatures.add(inferiorChainSignature); - - LOGGER.debug(() -> String.format("Synchronized with peer %s (%s)", peer, syncResult.name())); - break; - } - } - - // Has our chain tip changed? - BlockData newChainTip; - - try (final Repository repository = RepositoryManager.getRepository()) { - newChainTip = repository.getBlockRepository().getLastBlock(); - } catch (DataException e) { - LOGGER.warn(String.format("Repository issue when trying to fetch post-synchronization chain tip: %s", e.getMessage())); - return syncResult; - } + /** + * Export current trade bot states and minting accounts. + */ + public void exportRepositoryData() { + try (final Repository repository = RepositoryManager.getRepository()) { + repository.exportNodeLocalData(); - if (!Arrays.equals(newChainTip.getSignature(), priorChainTip.getSignature())) { - // Reset our cache of inferior chains - inferiorChainSignatures.clear(); + } catch (DataException e) { + // Fail silently as this is an optional step + } + } - // Update chain-tip, notify peers, websockets, etc. - this.onNewBlock(newChainTip); - } - return syncResult; - } finally { - isSynchronizing = false; - requestSysTrayUpdate = true; + public static class StatusChangeEvent implements Event { + public StatusChangeEvent() { } } - private void updateSysTray() { + public void updateSysTray() { if (NTP.getTime() == null) { SysTray.getInstance().setToolTipText(Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_CLOCK")); + SysTray.getInstance().setTrayIcon(1); return; } @@ -636,20 +761,31 @@ private void updateSysTray() { String heightText = Translator.INSTANCE.translate("SysTray", "BLOCK_HEIGHT"); String actionText; - if (isMintingPossible) - actionText = Translator.INSTANCE.translate("SysTray", "MINTING_ENABLED"); - else if (isSynchronizing) - actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), syncPercent); - else if (numberOfPeers < Settings.getInstance().getMinBlockchainPeers()) - actionText = Translator.INSTANCE.translate("SysTray", "CONNECTING"); - else - actionText = Translator.INSTANCE.translate("SysTray", "MINTING_DISABLED"); - String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height); + synchronized (Synchronizer.getInstance().syncLock) { + if (this.isMintingPossible) { + actionText = Translator.INSTANCE.translate("SysTray", "MINTING_ENABLED"); + SysTray.getInstance().setTrayIcon(2); + } + else if (numberOfPeers < Settings.getInstance().getMinBlockchainPeers()) { + actionText = Translator.INSTANCE.translate("SysTray", "CONNECTING"); + SysTray.getInstance().setTrayIcon(3); + } + else if (!this.isUpToDate()) { + actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), Synchronizer.getInstance().getSyncPercent()); + SysTray.getInstance().setTrayIcon(3); + } + else { + actionText = Translator.INSTANCE.translate("SysTray", "MINTING_DISABLED"); + SysTray.getInstance().setTrayIcon(4); + } + } + + String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height) + "\n" + String.format("%s: %s", Translator.INSTANCE.translate("SysTray", "BUILD_VERSION"), this.buildVersion); SysTray.getInstance().setToolTipText(tooltip); this.callbackExecutor.execute(() -> { - StatusNotifier.getInstance().onStatusChange(NTP.getTime()); + EventBus.INSTANCE.notify(new StatusChangeEvent()); }); } @@ -665,18 +801,144 @@ public void deleteExpiredTransactions() { List transactions = repository.getTransactionRepository().getUnconfirmedTransactions(); - for (TransactionData transactionData : transactions) - if (now >= Transaction.getDeadline(transactionData)) { - LOGGER.info(String.format("Deleting expired, unconfirmed transaction %s", Base58.encode(transactionData.getSignature()))); + int deletedCount = 0; + for (TransactionData transactionData : transactions) { + Transaction transaction = Transaction.fromData(repository, transactionData); + + if (now >= transaction.getDeadline()) { + LOGGER.debug(() -> String.format("Deleting expired, unconfirmed transaction %s", Base58.encode(transactionData.getSignature()))); repository.getTransactionRepository().delete(transactionData); + deletedCount++; } + } + if (deletedCount > 0) { + LOGGER.info(String.format("Deleted %d expired, unconfirmed transaction%s", deletedCount, (deletedCount == 1 ? "" : "s"))); + } repository.saveChanges(); } catch (DataException e) { - LOGGER.error("Repository issue while deleting expired unconfirmed transactions", e); + if (RepositoryManager.isDeadlockRelated(e)) + LOGGER.info("Couldn't delete some expired, unconfirmed transactions this round"); + else + LOGGER.error("Repository issue while deleting expired unconfirmed transactions", e); } } + // Incoming transactions queue + + private boolean incomingTransactionQueueContains(byte[] signature) { + synchronized (incomingTransactions) { + return incomingTransactions.stream().anyMatch(t -> Arrays.equals(t.getSignature(), signature)); + } + } + + private void removeIncomingTransaction(byte[] signature) { + incomingTransactions.removeIf(t -> Arrays.equals(t.getSignature(), signature)); + } + + private void processIncomingTransactionsQueue() { + if (this.incomingTransactions.size() == 0) { + // Don't bother locking if there are no new transactions to process + return; + } + + if (Synchronizer.getInstance().isSyncRequested() || Synchronizer.getInstance().isSynchronizing()) { + // Prioritize syncing, and don't attempt to lock + return; + } + + try { + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + if (!blockchainLock.tryLock(2, TimeUnit.SECONDS)) { + LOGGER.trace(() -> String.format("Too busy to process incoming transactions queue")); + return; + } + } catch (InterruptedException e) { + LOGGER.info("Interrupted when trying to acquire blockchain lock"); + return; + } + + try (final Repository repository = RepositoryManager.getRepository()) { + LOGGER.debug("Processing incoming transactions queue (size {})...", this.incomingTransactions.size()); + + // Take a copy of incomingTransactions so we can release the lock + ListincomingTransactionsCopy = new ArrayList<>(this.incomingTransactions); + + // Iterate through incoming transactions list + Iterator iterator = incomingTransactionsCopy.iterator(); + while (iterator.hasNext()) { + if (isStopping) { + return; + } + + if (Synchronizer.getInstance().isSyncRequestPending()) { + LOGGER.debug("Breaking out of transaction processing loop with {} remaining, because a sync request is pending", incomingTransactionsCopy.size()); + return; + } + + TransactionData transactionData = (TransactionData) iterator.next(); + Transaction transaction = Transaction.fromData(repository, transactionData); + + // Check signature + if (!transaction.isSignatureValid()) { + LOGGER.trace(() -> String.format("Ignoring %s transaction %s with invalid signature", transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); + removeIncomingTransaction(transactionData.getSignature()); + continue; + } + + ValidationResult validationResult = transaction.importAsUnconfirmed(); + + if (validationResult == ValidationResult.TRANSACTION_ALREADY_EXISTS) { + LOGGER.trace(() -> String.format("Ignoring existing transaction %s", Base58.encode(transactionData.getSignature()))); + removeIncomingTransaction(transactionData.getSignature()); + continue; + } + + if (validationResult == ValidationResult.NO_BLOCKCHAIN_LOCK) { + LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction", Base58.encode(transactionData.getSignature()))); + removeIncomingTransaction(transactionData.getSignature()); + continue; + } + + if (validationResult != ValidationResult.OK) { + final String signature58 = Base58.encode(transactionData.getSignature()); + LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58)); + Long now = NTP.getTime(); + if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) { + Long expiryLength = INVALID_TRANSACTION_RECHECK_INTERVAL; + if (validationResult == ValidationResult.TIMESTAMP_TOO_OLD) { + // Use shorter recheck interval for expired transactions + expiryLength = EXPIRED_TRANSACTION_RECHECK_INTERVAL; + } + Long expiry = now + expiryLength; + LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58); + // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it + invalidUnconfirmedTransactions.put(signature58, expiry); + } + removeIncomingTransaction(transactionData.getSignature()); + continue; + } + + LOGGER.debug(() -> String.format("Imported %s transaction %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); + removeIncomingTransaction(transactionData.getSignature()); + } + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while processing incoming transactions", e)); + } finally { + LOGGER.debug("Finished processing incoming transactions queue"); + blockchainLock.unlock(); + } + } + + private void cleanupInvalidTransactionsList(Long now) { + if (now == null) { + return; + } + // Periodically remove invalid unconfirmed transactions from the list, so that they can be fetched again + invalidUnconfirmedTransactions.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < now); + } + + // Shutdown public void shutdown() { @@ -684,6 +946,9 @@ public void shutdown() { if (!isStopping) { isStopping = true; + LOGGER.info("Shutting down synchronizer"); + Synchronizer.getInstance().shutdown(); + LOGGER.info("Shutting down API"); ApiService.getInstance().stop(); @@ -692,9 +957,14 @@ public void shutdown() { AutoUpdate.getInstance().shutdown(); } - // Arbitrary transaction data manager - // LOGGER.info("Shutting down arbitrary-transaction data manager"); - // ArbitraryDataManager.getInstance().shutdown(); + // Arbitrary data controllers + LOGGER.info("Shutting down arbitrary-transaction controllers"); + ArbitraryDataManager.getInstance().shutdown(); + ArbitraryDataFileManager.getInstance().shutdown(); + ArbitraryDataBuildManager.getInstance().shutdown(); + ArbitraryDataCleanupManager.getInstance().shutdown(); + ArbitraryDataStorageManager.getInstance().shutdown(); + ArbitraryDataRenderManager.getInstance().shutdown(); if (blockMinter != null) { LOGGER.info("Shutting down block minter"); @@ -706,6 +976,10 @@ public void shutdown() { } } + // Export local data + LOGGER.info("Backing up local data"); + this.exportRepositoryData(); + LOGGER.info("Shutting down networking"); Network.getInstance().shutdown(); @@ -717,6 +991,17 @@ public void shutdown() { // We were interrupted while waiting for thread to join } + // Make sure we're the only thread modifying the blockchain when shutting down the repository + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + try { + if (!blockchainLock.tryLock(5, TimeUnit.SECONDS)) { + LOGGER.debug("Couldn't acquire blockchain lock even after waiting 5 seconds"); + // Proceed anyway, as we have to shut down + } + } catch (InterruptedException e) { + LOGGER.info("Interrupted when waiting for blockchain lock"); + } + try { LOGGER.info("Shutting down repository"); RepositoryManager.closeRepositoryFactory(); @@ -724,6 +1009,11 @@ public void shutdown() { LOGGER.error("Error occurred while shutting down repository", e); } + // Release the lock if we acquired it + if (blockchainLock.isHeldByCurrentThread()) { + blockchainLock.unlock(); + } + LOGGER.info("Shutting down NTP"); NTP.shutdownNow(); @@ -765,40 +1055,165 @@ public void doNetworkBroadcast() { BlockData latestBlockData = getChainTip(); network.broadcast(peer -> network.buildHeightMessage(peer, latestBlockData)); - // Request unconfirmed transaction signatures, but only if we're up-to-date. - // If we're NOT up-to-date then priority is synchronizing first - if (isUpToDate()) - network.broadcast(network::buildGetUnconfirmedTransactionsMessage); - } + // Request unconfirmed transaction signatures, but only if we're up-to-date. + // If we're NOT up-to-date then priority is synchronizing first + if (isUpToDate()) + network.broadcast(network::buildGetUnconfirmedTransactionsMessage); + } + + public void onMintingPossibleChange(boolean isMintingPossible) { + this.isMintingPossible = isMintingPossible; + requestSysTrayUpdate = true; + } + + public static class NewBlockEvent implements Event { + private final BlockData blockData; + + public NewBlockEvent(BlockData blockData) { + this.blockData = blockData; + } + + public BlockData getBlockData() { + return this.blockData; + } + } + + /** + * Callback for when we've received a new block. + *

+ * See WARNING for {@link EventBus#notify(Event)} + * to prevent deadlocks. + */ + public void onNewBlock(BlockData latestBlockData) { + // Protective copy + BlockData blockDataCopy = new BlockData(latestBlockData); + int blockCacheSize = Settings.getInstance().getBlockCacheSize(); + + synchronized (this.latestBlocks) { + BlockData cachedChainTip = this.latestBlocks.peekLast(); + + if (cachedChainTip != null && Arrays.equals(cachedChainTip.getSignature(), blockDataCopy.getReference())) { + // Chain tip is parent for new latest block, so we can safely add new latest block + this.latestBlocks.addLast(latestBlockData); + + // Trim if necessary + if (this.latestBlocks.size() >= blockCacheSize) + this.latestBlocks.pollFirst(); + } else { + if (cachedChainTip != null) + // Chain tip didn't match - potentially abnormal behaviour? + LOGGER.debug(() -> String.format("Cached chain tip %.8s not parent for new latest block %.8s (reference %.8s)", + Base58.encode(cachedChainTip.getSignature()), + Base58.encode(blockDataCopy.getSignature()), + Base58.encode(blockDataCopy.getReference()))); + + // Defensively rebuild cache + try { + this.stats.latestBlocksCacheRefills.incrementAndGet(); + + this.refillLatestBlocksCache(); + } catch (DataException e) { + LOGGER.warn(() -> "Couldn't refill latest blocks cache?", e); + } + } + } + + this.onNewOrOrphanedBlock(blockDataCopy, NewBlockEvent::new); + } + + public static class OrphanedBlockEvent implements Event { + private final BlockData blockData; + + public OrphanedBlockEvent(BlockData blockData) { + this.blockData = blockData; + } + + public BlockData getBlockData() { + return this.blockData; + } + } + + /** + * Callback for when we've orphaned a block. + *

+ * See WARNING for {@link EventBus#notify(Event)} + * to prevent deadlocks. + */ + public void onOrphanedBlock(BlockData latestBlockData) { + // Protective copy + BlockData blockDataCopy = new BlockData(latestBlockData); + + synchronized (this.latestBlocks) { + BlockData cachedChainTip = this.latestBlocks.pollLast(); + boolean refillNeeded = false; + + if (cachedChainTip != null && Arrays.equals(cachedChainTip.getReference(), blockDataCopy.getSignature())) { + // Chain tip was parent for new latest block that has been orphaned, so we're good + + // However, if we've emptied the cache then we will need to refill it + refillNeeded = this.latestBlocks.isEmpty(); + } else { + if (cachedChainTip != null) + // Chain tip didn't match - potentially abnormal behaviour? + LOGGER.debug(() -> String.format("Cached chain tip %.8s (reference %.8s) was not parent for new latest block %.8s", + Base58.encode(cachedChainTip.getSignature()), + Base58.encode(cachedChainTip.getReference()), + Base58.encode(blockDataCopy.getSignature()))); + + // Defensively rebuild cache + refillNeeded = true; + } + + if (refillNeeded) + try { + this.stats.latestBlocksCacheRefills.incrementAndGet(); + + this.refillLatestBlocksCache(); + } catch (DataException e) { + LOGGER.warn(() -> "Couldn't refill latest blocks cache?", e); + } + } - public void onMintingPossibleChange(boolean isMintingPossible) { - this.isMintingPossible = isMintingPossible; - requestSysTrayUpdate = true; + this.onNewOrOrphanedBlock(blockDataCopy, OrphanedBlockEvent::new); } - public void onNewBlock(BlockData latestBlockData) { - this.setChainTip(latestBlockData); + private void onNewOrOrphanedBlock(BlockData blockDataCopy, Function eventConstructor) { requestSysTrayUpdate = true; - // Broadcast our new height info and notify websocket listeners - this.callbackExecutor.execute(() -> { - Network network = Network.getInstance(); - network.broadcast(peer -> network.buildHeightMessage(peer, latestBlockData)); + // Notify listeners, trade-bot, etc. + EventBus.INSTANCE.notify(eventConstructor.apply(blockDataCopy)); + + if (this.notifyGroupMembershipChange) { + this.notifyGroupMembershipChange = false; + ChatNotifier.getInstance().onGroupMembershipChange(); + } + } - BlockNotifier.getInstance().onNewBlock(latestBlockData); + public static class NewTransactionEvent implements Event { + private final TransactionData transactionData; - if (this.notifyGroupMembershipChange) { - this.notifyGroupMembershipChange = false; - ChatNotifier.getInstance().onGroupMembershipChange(); - } - }); + public NewTransactionEvent(TransactionData transactionData) { + this.transactionData = transactionData; + } + + public TransactionData getTransactionData() { + return this.transactionData; + } } - /** Callback for when we've received a new transaction via API or peer. */ - public void onNewTransaction(TransactionData transactionData, Peer peer) { + /** + * Callback for when we've received a new transaction via API or peer. + *

+ * @implSpec performs actions in a new thread + */ + public void onNewTransaction(TransactionData transactionData) { this.callbackExecutor.execute(() -> { - // Notify all peers (except maybe peer that sent it to us if applicable) - Network.getInstance().broadcast(broadcastPeer -> broadcastPeer == peer ? null : new TransactionSignaturesMessage(Arrays.asList(transactionData.getSignature()))); + // Notify all peers + Message newTransactionSignatureMessage = new TransactionSignaturesMessage(Arrays.asList(transactionData.getSignature())); + Network.getInstance().broadcast(broadcastPeer -> newTransactionSignatureMessage); + + // Notify listeners + EventBus.INSTANCE.notify(new NewTransactionEvent(transactionData)); // If this is a CHAT transaction, there may be extra listeners to notify if (transactionData.getType() == TransactionType.CHAT) @@ -861,20 +1276,40 @@ public void onNetworkMessage(Peer peer, Message message) { onNetworkTransactionSignaturesMessage(peer, message); break; + case GET_ONLINE_ACCOUNTS: + onNetworkGetOnlineAccountsMessage(peer, message); + break; + + case ONLINE_ACCOUNTS: + onNetworkOnlineAccountsMessage(peer, message); + break; + + case GET_ONLINE_ACCOUNTS_V2: + onNetworkGetOnlineAccountsV2Message(peer, message); + break; + + case ONLINE_ACCOUNTS_V2: + onNetworkOnlineAccountsV2Message(peer, message); + break; + case GET_ARBITRARY_DATA: - onNetworkGetArbitraryDataMessage(peer, message); + // Not currently supported break; - case ARBITRARY_DATA: - onNetworkArbitraryDataMessage(peer, message); + case ARBITRARY_DATA_FILE_LIST: + ArbitraryDataFileListManager.getInstance().onNetworkArbitraryDataFileListMessage(peer, message); break; - case GET_ONLINE_ACCOUNTS: - onNetworkGetOnlineAccountsMessage(peer, message); + case GET_ARBITRARY_DATA_FILE: + ArbitraryDataFileManager.getInstance().onNetworkGetArbitraryDataFileMessage(peer, message); break; - case ONLINE_ACCOUNTS: - onNetworkOnlineAccountsMessage(peer, message); + case GET_ARBITRARY_DATA_FILE_LIST: + ArbitraryDataFileListManager.getInstance().onNetworkGetArbitraryDataFileListMessage(peer, message); + break; + + case ARBITRARY_SIGNATURES: + ArbitraryDataManager.getInstance().onNetworkArbitrarySignaturesMessage(peer, message); break; default: @@ -886,120 +1321,237 @@ public void onNetworkMessage(Peer peer, Message message) { private void onNetworkGetBlockMessage(Peer peer, Message message) { GetBlockMessage getBlockMessage = (GetBlockMessage) message; byte[] signature = getBlockMessage.getSignature(); + this.stats.getBlockMessageStats.requests.incrementAndGet(); - try (final Repository repository = RepositoryManager.getRepository()) { - BlockData blockData = repository.getBlockRepository().fromSignature(signature); - if (blockData == null) { - LOGGER.debug(() -> String.format("Ignoring GET_BLOCK request from peer %s for unknown block %s", peer, Base58.encode(signature))); - // Send no response at all??? - return; - } + ByteArray signatureAsByteArray = new ByteArray(signature); - Block block = new Block(repository, blockData); + CachedBlockMessage cachedBlockMessage = this.blockMessageCache.get(signatureAsByteArray); + int blockCacheSize = Settings.getInstance().getBlockCacheSize(); - Message blockMessage = new BlockMessage(block); - blockMessage.setId(message.getId()); - if (!peer.sendMessage(blockMessage)) + // Check cached latest block message + if (cachedBlockMessage != null) { + this.stats.getBlockMessageStats.cacheHits.incrementAndGet(); + + // We need to duplicate it to prevent multiple threads setting ID on the same message + CachedBlockMessage clonedBlockMessage = cachedBlockMessage.cloneWithNewId(message.getId()); + + if (!peer.sendMessage(clonedBlockMessage)) peer.disconnect("failed to send block"); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while send block %s to peer %s", Base58.encode(signature), peer), e); - } - } - private void onNetworkTransactionMessage(Peer peer, Message message) { - TransactionMessage transactionMessage = (TransactionMessage) message; - TransactionData transactionData = transactionMessage.getTransactionData(); + return; + } try (final Repository repository = RepositoryManager.getRepository()) { - Transaction transaction = Transaction.fromData(repository, transactionData); + BlockData blockData = repository.getBlockRepository().fromSignature(signature); - // Check signature - if (!transaction.isSignatureValid()) { - LOGGER.trace(() -> String.format("Ignoring %s transaction %s with invalid signature from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer)); - return; + if (blockData != null) { + if (PruneManager.getInstance().isBlockPruned(blockData.getHeight())) { + // If this is a pruned block, we likely only have partial data, so best not to sent it + blockData = null; + } } - ValidationResult validationResult = transaction.importAsUnconfirmed(); - - if (validationResult == ValidationResult.TRANSACTION_ALREADY_EXISTS) { - LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(transactionData.getSignature()), peer)); - return; + // If we have no block data, we should check the archive in case it's there + if (blockData == null) { + if (Settings.getInstance().isArchiveEnabled()) { + byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, true, repository); + if (bytes != null) { + CachedBlockMessage blockMessage = new CachedBlockMessage(bytes); + blockMessage.setId(message.getId()); + + // This call also causes the other needed data to be pulled in from repository + if (!peer.sendMessage(blockMessage)) { + peer.disconnect("failed to send block"); + // Don't fall-through to caching because failure to send might be from failure to build message + return; + } + + // Sent successfully from archive, so nothing more to do + return; + } + } } - if (validationResult == ValidationResult.NO_BLOCKCHAIN_LOCK) { - LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction %s from peer %s", Base58.encode(transactionData.getSignature()), peer)); + if (blockData == null) { + // We don't have this block + this.stats.getBlockMessageStats.unknownBlocks.getAndIncrement(); + + // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout + LOGGER.debug(() -> String.format("Sending 'block unknown' response to peer %s for GET_BLOCK request for unknown block %s", peer, Base58.encode(signature))); + + // We'll send empty block summaries message as it's very short + Message blockUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + blockUnknownMessage.setId(message.getId()); + if (!peer.sendMessage(blockUnknownMessage)) + peer.disconnect("failed to send block-unknown response"); return; } - if (validationResult != ValidationResult.OK) { - LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s from peer %s", validationResult.name(), transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer)); + Block block = new Block(repository, blockData); + + CachedBlockMessage blockMessage = new CachedBlockMessage(block); + blockMessage.setId(message.getId()); + + // This call also causes the other needed data to be pulled in from repository + if (!peer.sendMessage(blockMessage)) { + peer.disconnect("failed to send block"); + // Don't fall-through to caching because failure to send might be from failure to build message return; } - LOGGER.debug(() -> String.format("Imported %s transaction %s from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer)); + // If request is for a recent block, cache it + if (getChainHeight() - blockData.getHeight() <= blockCacheSize) { + this.stats.getBlockMessageStats.cacheFills.incrementAndGet(); + + this.blockMessageCache.put(new ByteArray(blockData.getSignature()), blockMessage); + } } catch (DataException e) { - LOGGER.error(String.format("Repository issue while processing transaction %s from peer %s", Base58.encode(transactionData.getSignature()), peer), e); + LOGGER.error(String.format("Repository issue while send block %s to peer %s", Base58.encode(signature), peer), e); } + } - // Notify controller so it can notify other peers, etc. - Controller.getInstance().onNewTransaction(transactionData, peer); + private void onNetworkTransactionMessage(Peer peer, Message message) { + TransactionMessage transactionMessage = (TransactionMessage) message; + TransactionData transactionData = transactionMessage.getTransactionData(); + if (this.incomingTransactions.size() < MAX_INCOMING_TRANSACTIONS) { + if (!this.incomingTransactions.contains(transactionData)) { + this.incomingTransactions.add(transactionData); + } + } } private void onNetworkGetBlockSummariesMessage(Peer peer, Message message) { GetBlockSummariesMessage getBlockSummariesMessage = (GetBlockSummariesMessage) message; - byte[] parentSignature = getBlockSummariesMessage.getParentSignature(); + final byte[] parentSignature = getBlockSummariesMessage.getParentSignature(); + this.stats.getBlockSummariesStats.requests.incrementAndGet(); + + // If peer's parent signature matches our latest block signature + // then we can short-circuit with an empty response + BlockData chainTip = getChainTip(); + if (chainTip != null && Arrays.equals(parentSignature, chainTip.getSignature())) { + Message blockSummariesMessage = new BlockSummariesMessage(Collections.emptyList()); + blockSummariesMessage.setId(message.getId()); + if (!peer.sendMessage(blockSummariesMessage)) + peer.disconnect("failed to send block summaries"); - try (final Repository repository = RepositoryManager.getRepository()) { - List blockSummaries = new ArrayList<>(); + return; + } + + List blockSummaries = new ArrayList<>(); - int numberRequested = Math.min(Network.MAX_BLOCK_SUMMARIES_PER_REPLY, getBlockSummariesMessage.getNumberRequested()); + // Attempt to serve from our cache of latest blocks + synchronized (this.latestBlocks) { + blockSummaries = this.latestBlocks.stream() + .dropWhile(cachedBlockData -> !Arrays.equals(cachedBlockData.getReference(), parentSignature)) + .map(BlockSummaryData::new) + .collect(Collectors.toList()); + } + + if (blockSummaries.isEmpty()) { + try (final Repository repository = RepositoryManager.getRepository()) { + int numberRequested = Math.min(Network.MAX_BLOCK_SUMMARIES_PER_REPLY, getBlockSummariesMessage.getNumberRequested()); - do { BlockData blockData = repository.getBlockRepository().fromReference(parentSignature); + if (blockData == null) { + // Try the archive + blockData = repository.getBlockArchiveRepository().fromReference(parentSignature); + } - if (blockData == null) - // No more blocks to send to peer - break; + if (blockData != null) { + if (PruneManager.getInstance().isBlockPruned(blockData.getHeight())) { + // If this request contains a pruned block, we likely only have partial data, so best not to sent anything + // We always prune from the oldest first, so it's fine to just check the first block requested + blockData = null; + } + } - BlockSummaryData blockSummary = new BlockSummaryData(blockData); - blockSummaries.add(blockSummary); - parentSignature = blockData.getSignature(); - } while (blockSummaries.size() < numberRequested); + while (blockData != null && blockSummaries.size() < numberRequested) { + BlockSummaryData blockSummary = new BlockSummaryData(blockData); + blockSummaries.add(blockSummary); - Message blockSummariesMessage = new BlockSummariesMessage(blockSummaries); - blockSummariesMessage.setId(message.getId()); - if (!peer.sendMessage(blockSummariesMessage)) - peer.disconnect("failed to send block summaries"); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while sending block summaries after %s to peer %s", Base58.encode(parentSignature), peer), e); + byte[] previousSignature = blockData.getSignature(); + blockData = repository.getBlockRepository().fromReference(previousSignature); + if (blockData == null) { + // Try the archive + blockData = repository.getBlockArchiveRepository().fromReference(previousSignature); + } + } + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while sending block summaries after %s to peer %s", Base58.encode(parentSignature), peer), e); + } + } else { + this.stats.getBlockSummariesStats.cacheHits.incrementAndGet(); + + if (blockSummaries.size() >= getBlockSummariesMessage.getNumberRequested()) + this.stats.getBlockSummariesStats.fullyFromCache.incrementAndGet(); } + + Message blockSummariesMessage = new BlockSummariesMessage(blockSummaries); + blockSummariesMessage.setId(message.getId()); + if (!peer.sendMessage(blockSummariesMessage)) + peer.disconnect("failed to send block summaries"); } private void onNetworkGetSignaturesV2Message(Peer peer, Message message) { GetSignaturesV2Message getSignaturesMessage = (GetSignaturesV2Message) message; - byte[] parentSignature = getSignaturesMessage.getParentSignature(); + final byte[] parentSignature = getSignaturesMessage.getParentSignature(); + this.stats.getBlockSignaturesV2Stats.requests.incrementAndGet(); + + // If peer's parent signature matches our latest block signature + // then we can short-circuit with an empty response + BlockData chainTip = getChainTip(); + if (chainTip != null && Arrays.equals(parentSignature, chainTip.getSignature())) { + Message signaturesMessage = new SignaturesMessage(Collections.emptyList()); + signaturesMessage.setId(message.getId()); + if (!peer.sendMessage(signaturesMessage)) + peer.disconnect("failed to send signatures (v2)"); - try (final Repository repository = RepositoryManager.getRepository()) { - List signatures = new ArrayList<>(); + return; + } + + List signatures = new ArrayList<>(); + + // Attempt to serve from our cache of latest blocks + synchronized (this.latestBlocks) { + signatures = this.latestBlocks.stream() + .dropWhile(cachedBlockData -> !Arrays.equals(cachedBlockData.getReference(), parentSignature)) + .map(BlockData::getSignature) + .collect(Collectors.toList()); + } - do { + if (signatures.isEmpty()) { + try (final Repository repository = RepositoryManager.getRepository()) { + int numberRequested = getSignaturesMessage.getNumberRequested(); BlockData blockData = repository.getBlockRepository().fromReference(parentSignature); + if (blockData == null) { + // Try the archive + blockData = repository.getBlockArchiveRepository().fromReference(parentSignature); + } - if (blockData == null) - // No more signatures to send to peer - break; + while (blockData != null && signatures.size() < numberRequested) { + signatures.add(blockData.getSignature()); - parentSignature = blockData.getSignature(); - signatures.add(parentSignature); - } while (signatures.size() < getSignaturesMessage.getNumberRequested()); + byte[] previousSignature = blockData.getSignature(); + blockData = repository.getBlockRepository().fromReference(previousSignature); + if (blockData == null) { + // Try the archive + blockData = repository.getBlockArchiveRepository().fromReference(previousSignature); + } + } + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while sending V2 signatures after %s to peer %s", Base58.encode(parentSignature), peer), e); + } + } else { + this.stats.getBlockSignaturesV2Stats.cacheHits.incrementAndGet(); - Message signaturesMessage = new SignaturesMessage(signatures); - signaturesMessage.setId(message.getId()); - if (!peer.sendMessage(signaturesMessage)) - peer.disconnect("failed to send signatures (v2)"); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while sending V2 signatures after %s to peer %s", Base58.encode(parentSignature), peer), e); + if (signatures.size() >= getSignaturesMessage.getNumberRequested()) + this.stats.getBlockSignaturesV2Stats.fullyFromCache.incrementAndGet(); } + + Message signaturesMessage = new SignaturesMessage(signatures); + signaturesMessage.setId(message.getId()); + if (!peer.sendMessage(signaturesMessage)) + peer.disconnect("failed to send signatures (v2)"); } private void onNetworkHeightV2Message(Peer peer, Message message) { @@ -1016,7 +1568,7 @@ private void onNetworkHeightV2Message(Peer peer, Message message) { peer.setChainTipData(newChainTipData); // Potentially synchronize - requestSync = true; + Synchronizer.getInstance().requestSync(); } private void onNetworkGetTransactionMessage(Peer peer, Message message) { @@ -1063,6 +1615,19 @@ private void onNetworkTransactionSignaturesMessage(Peer peer, Message message) { try (final Repository repository = RepositoryManager.getRepository()) { for (byte[] signature : signatures) { + String signature58 = Base58.encode(signature); + if (invalidUnconfirmedTransactions.containsKey(signature58)) { + // Previously invalid transaction - don't keep requesting it + // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks + continue; + } + + // Ignore if this transaction is in the queue + if (incomingTransactionQueueContains(signature)) { + LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peer)); + continue; + } + // Do we have it already? (Before requesting transaction data itself) if (repository.getTransactionRepository().exists(signature)) { LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(signature), peer)); @@ -1085,105 +1650,55 @@ private void onNetworkTransactionSignaturesMessage(Peer peer, Message message) { } } - private void onNetworkGetArbitraryDataMessage(Peer peer, Message message) { - GetArbitraryDataMessage getArbitraryDataMessage = (GetArbitraryDataMessage) message; - - byte[] signature = getArbitraryDataMessage.getSignature(); - String signature58 = Base58.encode(signature); - Long timestamp = NTP.getTime(); - Triple newEntry = new Triple<>(signature58, peer, timestamp); - - // If we've seen this request recently, then ignore - if (arbitraryDataRequests.putIfAbsent(message.getId(), newEntry) != null) - return; + private void onNetworkGetOnlineAccountsMessage(Peer peer, Message message) { + GetOnlineAccountsMessage getOnlineAccountsMessage = (GetOnlineAccountsMessage) message; - // Do we even have this transaction? - try (final Repository repository = RepositoryManager.getRepository()) { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (transactionData == null || transactionData.getType() != TransactionType.ARBITRARY) - return; + List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); - ArbitraryTransaction transaction = new ArbitraryTransaction(repository, transactionData); + // Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts + List accountsToSend; + synchronized (this.onlineAccounts) { + accountsToSend = new ArrayList<>(this.onlineAccounts); + } - // If we have the data then send it - if (transaction.isDataLocal()) { - byte[] data = transaction.fetchData(); - if (data == null) - return; + Iterator iterator = accountsToSend.iterator(); - // Update requests map to reflect that we've sent it - newEntry = new Triple<>(signature58, null, timestamp); - arbitraryDataRequests.put(message.getId(), newEntry); + SEND_ITERATOR: + while (iterator.hasNext()) { + OnlineAccountData onlineAccountData = iterator.next(); - Message arbitraryDataMessage = new ArbitraryDataMessage(signature, data); - arbitraryDataMessage.setId(message.getId()); - if (!peer.sendMessage(arbitraryDataMessage)) - peer.disconnect("failed to send arbitrary data"); + for (int i = 0; i < excludeAccounts.size(); ++i) { + OnlineAccountData excludeAccountData = excludeAccounts.get(i); - return; + if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) { + iterator.remove(); + continue SEND_ITERATOR; + } } - - // Ask our other peers if they have it - Network.getInstance().broadcast(broadcastPeer -> broadcastPeer == peer ? null : message); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while finding arbitrary transaction data for peer %s", peer), e); } - } - private void onNetworkArbitraryDataMessage(Peer peer, Message message) { - ArbitraryDataMessage arbitraryDataMessage = (ArbitraryDataMessage) message; + Message onlineAccountsMessage = new OnlineAccountsMessage(accountsToSend); + peer.sendMessage(onlineAccountsMessage); - // Do we have a pending request for this data? - Triple request = arbitraryDataRequests.get(message.getId()); - if (request == null || request.getA() == null) - return; + LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer)); + } - // Does this message's signature match what we're expecting? - byte[] signature = arbitraryDataMessage.getSignature(); - String signature58 = Base58.encode(signature); - if (!request.getA().equals(signature58)) - return; + private void onNetworkOnlineAccountsMessage(Peer peer, Message message) { + OnlineAccountsMessage onlineAccountsMessage = (OnlineAccountsMessage) message; - byte[] data = arbitraryDataMessage.getData(); + List peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts(); + LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer)); - // Check transaction exists and payload hash is correct try (final Repository repository = RepositoryManager.getRepository()) { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (!(transactionData instanceof ArbitraryTransactionData)) - return; - - ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - - byte[] actualHash = Crypto.digest(data); - - // "data" from repository will always be hash of actual raw data - if (!Arrays.equals(arbitraryTransactionData.getData(), actualHash)) - return; - - // Update requests map to reflect that we've received it - Triple newEntry = new Triple<>(null, null, request.getC()); - arbitraryDataRequests.put(message.getId(), newEntry); - - // Save payload locally - // TODO: storage policy - arbitraryTransactionData.setDataType(DataType.RAW_DATA); - arbitraryTransactionData.setData(data); - repository.getArbitraryRepository().save(arbitraryTransactionData); - repository.saveChanges(); + for (OnlineAccountData onlineAccountData : peersOnlineAccounts) + this.verifyAndAddAccount(repository, onlineAccountData); } catch (DataException e) { - LOGGER.error(String.format("Repository issue while finding arbitrary transaction data for peer %s", peer), e); - } - - Peer requestingPeer = request.getB(); - if (requestingPeer != null) { - // Forward to requesting peer; - if (!requestingPeer.sendMessage(arbitraryDataMessage)) - requestingPeer.disconnect("failed to forward arbitrary data"); + LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e); } } - private void onNetworkGetOnlineAccountsMessage(Peer peer, Message message) { - GetOnlineAccountsMessage getOnlineAccountsMessage = (GetOnlineAccountsMessage) message; + private void onNetworkGetOnlineAccountsV2Message(Peer peer, Message message) { + GetOnlineAccountsV2Message getOnlineAccountsMessage = (GetOnlineAccountsV2Message) message; List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); @@ -1209,14 +1724,14 @@ private void onNetworkGetOnlineAccountsMessage(Peer peer, Message message) { } } - Message onlineAccountsMessage = new OnlineAccountsMessage(accountsToSend); + Message onlineAccountsMessage = new OnlineAccountsV2Message(accountsToSend); peer.sendMessage(onlineAccountsMessage); LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer)); } - private void onNetworkOnlineAccountsMessage(Peer peer, Message message) { - OnlineAccountsMessage onlineAccountsMessage = (OnlineAccountsMessage) message; + private void onNetworkOnlineAccountsV2Message(Peer peer, Message message) { + OnlineAccountsV2Message onlineAccountsMessage = (OnlineAccountsV2Message) message; List peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts(); LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer)); @@ -1340,104 +1855,117 @@ private void performOnlineAccountsTasks() { // Request data from other peers? if ((this.onlineAccountsTasksTimestamp % ONLINE_ACCOUNTS_BROADCAST_INTERVAL) < ONLINE_ACCOUNTS_TASKS_INTERVAL) { - Message message; + List safeOnlineAccounts; synchronized (this.onlineAccounts) { - message = new GetOnlineAccountsMessage(this.onlineAccounts); + safeOnlineAccounts = new ArrayList<>(this.onlineAccounts); } - Network.getInstance().broadcast(peer -> message); + + Message messageV1 = new GetOnlineAccountsMessage(safeOnlineAccounts); + Message messageV2 = new GetOnlineAccountsV2Message(safeOnlineAccounts); + + Network.getInstance().broadcast(peer -> + peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1 + ); } // Refresh our online accounts signatures? sendOurOnlineAccountsInfo(); - - // Trim blockchain by removing 'old' online accounts signatures - BlockChain.trimOldOnlineAccountsSignatures(); } private void sendOurOnlineAccountsInfo() { final Long now = NTP.getTime(); - if (now == null) - return; + if (now != null) { - List mintingAccounts; - try (final Repository repository = RepositoryManager.getRepository()) { - mintingAccounts = repository.getAccountRepository().getMintingAccounts(); + List mintingAccounts; + try (final Repository repository = RepositoryManager.getRepository()) { + mintingAccounts = repository.getAccountRepository().getMintingAccounts(); - // We have no accounts, but don't reset timestamp - if (mintingAccounts.isEmpty()) - return; + // We have no accounts, but don't reset timestamp + if (mintingAccounts.isEmpty()) + return; - // Only reward-share accounts allowed - Iterator iterator = mintingAccounts.iterator(); - while (iterator.hasNext()) { - MintingAccountData mintingAccountData = iterator.next(); + // Only reward-share accounts allowed + Iterator iterator = mintingAccounts.iterator(); + int i = 0; + while (iterator.hasNext()) { + MintingAccountData mintingAccountData = iterator.next(); - RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey()); - if (rewardShareData == null) { - // Reward-share doesn't even exist - probably not a good sign - iterator.remove(); - continue; - } + RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey()); + if (rewardShareData == null) { + // Reward-share doesn't even exist - probably not a good sign + iterator.remove(); + continue; + } - Account mintingAccount = new Account(repository, rewardShareData.getMinter()); - if (!mintingAccount.canMint()) { - // Minting-account component of reward-share can no longer mint - disregard - iterator.remove(); - continue; + Account mintingAccount = new Account(repository, rewardShareData.getMinter()); + if (!mintingAccount.canMint()) { + // Minting-account component of reward-share can no longer mint - disregard + iterator.remove(); + continue; + } + + if (++i > 2) { + iterator.remove(); + continue; + } } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage())); + return; } - } catch (DataException e) { - LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage())); - return; - } - - // 'current' timestamp - final long onlineAccountsTimestamp = Controller.toOnlineAccountTimestamp(now); - boolean hasInfoChanged = false; - - byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); - List ourOnlineAccounts = new ArrayList<>(); - MINTING_ACCOUNTS: - for (MintingAccountData mintingAccountData : mintingAccounts) { - PrivateKeyAccount mintingAccount = new PrivateKeyAccount(null, mintingAccountData.getPrivateKey()); + // 'current' timestamp + final long onlineAccountsTimestamp = Controller.toOnlineAccountTimestamp(now); + boolean hasInfoChanged = false; - byte[] signature = mintingAccount.sign(timestampBytes); - byte[] publicKey = mintingAccount.getPublicKey(); + byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); + List ourOnlineAccounts = new ArrayList<>(); - // Our account is online - OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey); - synchronized (this.onlineAccounts) { - Iterator iterator = this.onlineAccounts.iterator(); - while (iterator.hasNext()) { - OnlineAccountData existingOnlineAccountData = iterator.next(); + MINTING_ACCOUNTS: + for (MintingAccountData mintingAccountData : mintingAccounts) { + PrivateKeyAccount mintingAccount = new PrivateKeyAccount(null, mintingAccountData.getPrivateKey()); - if (Arrays.equals(existingOnlineAccountData.getPublicKey(), ourOnlineAccountData.getPublicKey())) { - // If our online account is already present, with same timestamp, then move on to next mintingAccount - if (existingOnlineAccountData.getTimestamp() == onlineAccountsTimestamp) - continue MINTING_ACCOUNTS; + byte[] signature = mintingAccount.sign(timestampBytes); + byte[] publicKey = mintingAccount.getPublicKey(); - // If our online account is already present, but with older timestamp, then remove it - iterator.remove(); - break; + // Our account is online + OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey); + synchronized (this.onlineAccounts) { + Iterator iterator = this.onlineAccounts.iterator(); + while (iterator.hasNext()) { + OnlineAccountData existingOnlineAccountData = iterator.next(); + + if (Arrays.equals(existingOnlineAccountData.getPublicKey(), ourOnlineAccountData.getPublicKey())) { + // If our online account is already present, with same timestamp, then move on to next mintingAccount + if (existingOnlineAccountData.getTimestamp() == onlineAccountsTimestamp) + continue MINTING_ACCOUNTS; + + // If our online account is already present, but with older timestamp, then remove it + iterator.remove(); + break; + } } + + this.onlineAccounts.add(ourOnlineAccountData); } - this.onlineAccounts.add(ourOnlineAccountData); + LOGGER.trace(() -> String.format("Added our online account %s with timestamp %d", mintingAccount.getAddress(), onlineAccountsTimestamp)); + ourOnlineAccounts.add(ourOnlineAccountData); + hasInfoChanged = true; } - LOGGER.trace(() -> String.format("Added our online account %s with timestamp %d", mintingAccount.getAddress(), onlineAccountsTimestamp)); - ourOnlineAccounts.add(ourOnlineAccountData); - hasInfoChanged = true; - } + if (!hasInfoChanged) + return; - if (!hasInfoChanged) - return; + Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts); + Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts); - Message message = new OnlineAccountsMessage(ourOnlineAccounts); - Network.getInstance().broadcast(peer -> message); + Network.getInstance().broadcast(peer -> + peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1 + ); - LOGGER.trace(()-> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp)); + LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp)); + } } public static long toOnlineAccountTimestamp(long timestamp) { @@ -1453,48 +1981,29 @@ public List getOnlineAccounts() { } } - public byte[] fetchArbitraryData(byte[] signature) throws InterruptedException { - // Build request - Message getArbitraryDataMessage = new GetArbitraryDataMessage(signature); - - // Save our request into requests map - String signature58 = Base58.encode(signature); - Triple requestEntry = new Triple<>(signature58, null, NTP.getTime()); - - // Assign random ID to this message - int id; - do { - id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1; - - // Put queue into map (keyed by message ID) so we can poll for a response - // If putIfAbsent() doesn't return null, then this ID is already taken - } while (arbitraryDataRequests.put(id, requestEntry) != null); - getArbitraryDataMessage.setId(id); - - // Broadcast request - Network.getInstance().broadcast(peer -> getArbitraryDataMessage); - - // Poll to see if data has arrived - final long singleWait = 100; - long totalWait = 0; - while (totalWait < ARBITRARY_REQUEST_TIMEOUT) { - Thread.sleep(singleWait); - - requestEntry = arbitraryDataRequests.get(id); - if (requestEntry == null) - return null; + /** Returns cached, unmodifiable list of latest block's online accounts. */ + public List getLatestBlocksOnlineAccounts() { + synchronized (this.latestBlocksOnlineAccounts) { + return this.latestBlocksOnlineAccounts.peekFirst(); + } + } - if (requestEntry.getA() == null) - break; + /** Caches list of latest block's online accounts. Typically called by Block.process() */ + public void pushLatestBlocksOnlineAccounts(List latestBlocksOnlineAccounts) { + synchronized (this.latestBlocksOnlineAccounts) { + if (this.latestBlocksOnlineAccounts.size() == MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS) + this.latestBlocksOnlineAccounts.pollLast(); - totalWait += singleWait; + this.latestBlocksOnlineAccounts.addFirst(latestBlocksOnlineAccounts == null + ? Collections.emptyList() + : Collections.unmodifiableList(latestBlocksOnlineAccounts)); } + } - try (final Repository repository = RepositoryManager.getRepository()) { - return repository.getArbitraryRepository().fetchData(signature); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while fetching arbitrary transaction data"), e); - return null; + /** Reverts list of latest block's online accounts. Typically called by Block.orphan() */ + public void popLatestBlocksOnlineAccounts() { + synchronized (this.latestBlocksOnlineAccounts) { + this.latestBlocksOnlineAccounts.pollFirst(); } } @@ -1587,4 +2096,8 @@ public static Long getMinimumLatestBlockTimestamp() { return now - offset; } + public StatsSnapshot getStatsSnapshot() { + return this.stats; + } + } diff --git a/src/main/java/org/qortal/controller/StatusNotifier.java b/src/main/java/org/qortal/controller/StatusNotifier.java deleted file mode 100644 index 02090db37..000000000 --- a/src/main/java/org/qortal/controller/StatusNotifier.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.qortal.controller; - -import java.util.HashMap; -import java.util.Map; - -import org.eclipse.jetty.websocket.api.Session; - -public class StatusNotifier { - - private static StatusNotifier instance; - - @FunctionalInterface - public interface Listener { - void notify(long timestamp); - } - - private Map listenersBySession = new HashMap<>(); - - private StatusNotifier() { - } - - public static synchronized StatusNotifier getInstance() { - if (instance == null) - instance = new StatusNotifier(); - - return instance; - } - - public synchronized void register(Session session, Listener listener) { - this.listenersBySession.put(session, listener); - } - - public synchronized void deregister(Session session) { - this.listenersBySession.remove(session); - } - - public synchronized void onStatusChange(long now) { - for (Listener listener : this.listenersBySession.values()) - listener.notify(now); - } - -} diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index e7dc6dd2b..b98c5fa25 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -1,22 +1,28 @@ package org.qortal.controller; import java.math.BigInteger; +import java.security.SecureRandom; import java.text.DecimalFormat; import java.text.NumberFormat; -import java.util.ArrayList; -import java.util.List; +import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.account.Account; +import org.qortal.account.PublicKeyAccount; import org.qortal.block.Block; import org.qortal.block.Block.ValidationResult; +import org.qortal.block.BlockChain; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; +import org.qortal.data.block.CommonBlockData; import org.qortal.data.network.PeerChainTipData; +import org.qortal.data.transaction.RewardShareTransactionData; import org.qortal.data.transaction.TransactionData; +import org.qortal.network.Network; import org.qortal.network.Peer; import org.qortal.network.message.BlockMessage; import org.qortal.network.message.BlockSummariesMessage; @@ -29,17 +35,60 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; import org.qortal.transaction.Transaction; import org.qortal.utils.Base58; +import org.qortal.utils.ByteArray; +import org.qortal.utils.NTP; -public class Synchronizer { +public class Synchronizer extends Thread { private static final Logger LOGGER = LogManager.getLogger(Synchronizer.class); + /** Max number of new blocks we aim to add to chain tip in each sync round */ + private static final int SYNC_BATCH_SIZE = 1000; // XXX move to Settings? + + /** Initial jump back of block height when searching for common block with peer */ private static final int INITIAL_BLOCK_STEP = 8; - private static final int MAXIMUM_BLOCK_STEP = 500; + /** Maximum jump back of block height when searching for common block with peer */ + private static final int MAXIMUM_BLOCK_STEP = 128; + + /** Maximum difference in block height between tip and peer's common block before peer is considered TOO DIVERGENT */ private static final int MAXIMUM_COMMON_DELTA = 240; // XXX move to Settings? - private static final int SYNC_BATCH_SIZE = 200; + + /** Maximum number of block signatures we ask from peer in one go */ + private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings? + + private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms + + + private boolean running; + + /** Latest block signatures from other peers that we know are on inferior chains. */ + List inferiorChainSignatures = new ArrayList<>(); + + /** Recovery mode, which is used to bring back a stalled network */ + private boolean recoveryMode = false; + private boolean peersAvailable = true; // peersAvailable must default to true + private long timePeersLastAvailable = 0; + + // Keep track of the size of the last re-org, so it can be logged + private int lastReorgSize; + + /** Synchronization object for sync variables below */ + public final Object syncLock = new Object(); + /** Whether we are attempting to synchronize. */ + private volatile boolean isSynchronizing = false; + /** Temporary estimate of synchronization progress for SysTray use. */ + private volatile int syncPercent = 0; + + private static volatile boolean requestSync = false; + private boolean syncRequestPending = false; + + // Keep track of invalid blocks so that we don't keep trying to sync them + private Map invalidBlockSignatures = Collections.synchronizedMap(new HashMap<>()); + public Long timeValidBlockLastReceived = null; + public Long timeInvalidBlockLastReceived = null; private static Synchronizer instance; @@ -50,6 +99,7 @@ public enum SynchronizationResult { // Constructors private Synchronizer() { + this.running = true; } public static Synchronizer getInstance() { @@ -59,6 +109,755 @@ public static Synchronizer getInstance() { return instance; } + + @Override + public void run() { + Thread.currentThread().setName("Synchronizer"); + + try { + while (running && !Controller.isStopping()) { + Thread.sleep(1000); + + if (requestSync) { + requestSync = false; + boolean success = Synchronizer.getInstance().potentiallySynchronize(); + if (!success) { + // Something went wrong, so try again next time + requestSync = true; + } + // Remember that we have a pending sync request if this attempt failed + syncRequestPending = !success; + } + } + } catch (InterruptedException e) { + // Clear interrupted flag so we can shutdown trim threads + Thread.interrupted(); + // Fall-through to exit + } + } + + public void shutdown() { + this.running = false; + this.interrupt(); + } + + + + public boolean isSynchronizing() { + return this.isSynchronizing; + } + + public boolean isSyncRequestPending() { + return this.syncRequestPending; + } + + public Integer getSyncPercent() { + synchronized (this.syncLock) { + return this.isSynchronizing ? this.syncPercent : null; + } + } + + public void requestSync() { + requestSync = true; + } + + public boolean isSyncRequested() { + return requestSync; + } + + public boolean getRecoveryMode() { + return this.recoveryMode; + } + + + public boolean potentiallySynchronize() throws InterruptedException { + // Already synchronizing via another thread? + if (this.isSynchronizing) + return true; + + List peers = Network.getInstance().getHandshakedPeers(); + + // Disregard peers that have "misbehaved" recently + peers.removeIf(Controller.hasMisbehaved); + + // Disregard peers that only have genesis block + peers.removeIf(Controller.hasOnlyGenesisBlock); + + // Disregard peers that don't have a recent block + peers.removeIf(Controller.hasNoRecentBlock); + + // Disregard peers that are on an old version + peers.removeIf(Controller.hasOldVersion); + + checkRecoveryModeForPeers(peers); + if (recoveryMode) { + peers = Network.getInstance().getHandshakedPeers(); + peers.removeIf(Controller.hasOnlyGenesisBlock); + peers.removeIf(Controller.hasMisbehaved); + peers.removeIf(Controller.hasOldVersion); + } + + // Check we have enough peers to potentially synchronize + if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) + return true; + + // Disregard peers that have no block signature or the same block signature as us + peers.removeIf(Controller.hasNoOrSameBlock); + + // Disregard peers that are on the same block as last sync attempt and we didn't like their chain + peers.removeIf(Controller.hasInferiorChainTip); + + final int peersBeforeComparison = peers.size(); + + // Request recent block summaries from the remaining peers, and locate our common block with each + Synchronizer.getInstance().findCommonBlocksWithPeers(peers); + + // Compare the peers against each other, and against our chain, which will return an updated list excluding those without common blocks + peers = Synchronizer.getInstance().comparePeers(peers); + + // We may have added more inferior chain tips when comparing peers, so remove any peers that are currently on those chains + peers.removeIf(Controller.hasInferiorChainTip); + + final int peersRemoved = peersBeforeComparison - peers.size(); + if (peersRemoved > 0 && peers.size() > 0) + LOGGER.debug(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size())); + + if (peers.isEmpty()) + return true; + + if (peers.size() > 1) { + StringBuilder finalPeersString = new StringBuilder(); + for (Peer peer : peers) + finalPeersString = finalPeersString.length() > 0 ? finalPeersString.append(", ").append(peer) : finalPeersString.append(peer); + LOGGER.debug(String.format("Choosing random peer from: [%s]", finalPeersString.toString())); + } + + // Pick random peer to sync with + int index = new SecureRandom().nextInt(peers.size()); + Peer peer = peers.get(index); + + SynchronizationResult syncResult = actuallySynchronize(peer, false); + if (syncResult == SynchronizationResult.NO_BLOCKCHAIN_LOCK) { + // No blockchain lock - force a retry by returning false + return false; + } + + return true; + } + + public SynchronizationResult actuallySynchronize(Peer peer, boolean force) throws InterruptedException { + boolean hasStatusChanged = false; + BlockData priorChainTip = Controller.getInstance().getChainTip(); + + synchronized (this.syncLock) { + this.syncPercent = (priorChainTip.getHeight() * 100) / peer.getChainTipData().getLastHeight(); + + // Only update SysTray if we're potentially changing height + if (this.syncPercent < 100) { + this.isSynchronizing = true; + hasStatusChanged = true; + } + } + peer.setSyncInProgress(true); + + if (hasStatusChanged) + Controller.getInstance().updateSysTray(); + + try { + SynchronizationResult syncResult = Synchronizer.getInstance().synchronize(peer, force); + switch (syncResult) { + case GENESIS_ONLY: + case NO_COMMON_BLOCK: + case TOO_DIVERGENT: + case INVALID_DATA: { + // These are more serious results that warrant a cool-off + LOGGER.info(String.format("Failed to synchronize with peer %s (%s) - cooling off", peer, syncResult.name())); + + // Don't use this peer again for a while + Network.getInstance().peerMisbehaved(peer); + break; + } + + case INFERIOR_CHAIN: { + // Update our list of inferior chain tips + ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature()); + if (!inferiorChainSignatures.contains(inferiorChainSignature)) + inferiorChainSignatures.add(inferiorChainSignature); + + // These are minor failure results so fine to try again + LOGGER.debug(() -> String.format("Refused to synchronize with peer %s (%s)", peer, syncResult.name())); + + // Notify peer of our superior chain + if (!peer.sendMessage(Network.getInstance().buildHeightMessage(peer, priorChainTip))) + peer.disconnect("failed to notify peer of our superior chain"); + break; + } + + case NO_REPLY: + case NO_BLOCKCHAIN_LOCK: + case REPOSITORY_ISSUE: + // These are minor failure results so fine to try again + LOGGER.debug(() -> String.format("Failed to synchronize with peer %s (%s)", peer, syncResult.name())); + break; + + case SHUTTING_DOWN: + // Just quietly exit + break; + + case OK: + // fall-through... + case NOTHING_TO_DO: { + // Update our list of inferior chain tips + ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature()); + if (!inferiorChainSignatures.contains(inferiorChainSignature)) + inferiorChainSignatures.add(inferiorChainSignature); + + LOGGER.debug(() -> String.format("Synchronized with peer %s (%s)", peer, syncResult.name())); + break; + } + } + + if (!running) { + // We've stopped + return SynchronizationResult.SHUTTING_DOWN; + } + + // Has our chain tip changed? + BlockData newChainTip; + + try (final Repository repository = RepositoryManager.getRepository()) { + newChainTip = repository.getBlockRepository().getLastBlock(); + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue when trying to fetch post-synchronization chain tip: %s", e.getMessage())); + return syncResult; + } + + if (!Arrays.equals(newChainTip.getSignature(), priorChainTip.getSignature())) { + // Reset our cache of inferior chains + inferiorChainSignatures.clear(); + + Network network = Network.getInstance(); + network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip)); + } + + return syncResult; + } finally { + this.isSynchronizing = false; + peer.setSyncInProgress(false); + } + } + + private boolean checkRecoveryModeForPeers(List qualifiedPeers) { + List handshakedPeers = Network.getInstance().getHandshakedPeers(); + + if (handshakedPeers.size() > 0) { + // There is at least one handshaked peer + if (qualifiedPeers.isEmpty()) { + // There are no 'qualified' peers - i.e. peers that have a recent block we can sync to + boolean werePeersAvailable = peersAvailable; + peersAvailable = false; + + // If peers only just became unavailable, update our record of the time they were last available + if (werePeersAvailable) + timePeersLastAvailable = NTP.getTime(); + + // If enough time has passed, enter recovery mode, which lifts some restrictions on who we can sync with and when we can mint + if (NTP.getTime() - timePeersLastAvailable > RECOVERY_MODE_TIMEOUT) { + if (recoveryMode == false) { + LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", RECOVERY_MODE_TIMEOUT/60/1000)); + recoveryMode = true; + } + } + } else { + // We now have at least one peer with a recent block, so we can exit recovery mode and sync normally + peersAvailable = true; + if (recoveryMode) { + LOGGER.info("Peers have become available again. Exiting recovery mode..."); + recoveryMode = false; + } + } + } + return recoveryMode; + } + + public void addInferiorChainSignature(byte[] inferiorSignature) { + // Update our list of inferior chain tips + ByteArray inferiorChainSignature = new ByteArray(inferiorSignature); + if (!inferiorChainSignatures.contains(inferiorChainSignature)) + inferiorChainSignatures.add(inferiorChainSignature); + } + + + /** + * Iterate through a list of supplied peers, and attempt to find our common block with each. + * If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later. + *

+ * Will return SynchronizationResult.OK on success. + *

+ * @param peers + * @return SynchronizationResult.OK if the process completed successfully, or a different SynchronizationResult if something went wrong. + * @throws InterruptedException + */ + public SynchronizationResult findCommonBlocksWithPeers(List peers) throws InterruptedException { + try (final Repository repository = RepositoryManager.getRepository()) { + try { + + if (peers.size() == 0) + return SynchronizationResult.NOTHING_TO_DO; + + // If our latest block is very old, it's best that we don't try and determine the best peers to sync to. + // This is because it can involve very large chain comparisons, which is too intensive. + // In reality, most forking problems occur near the chain tips, so we will reserve this functionality for those situations. + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null) + return SynchronizationResult.REPOSITORY_ISSUE; + + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + LOGGER.debug(String.format("Our latest block is very old, so we won't collect common block info from peers")); + return SynchronizationResult.NOTHING_TO_DO; + } + + LOGGER.debug(String.format("Searching for common blocks with %d peers...", peers.size())); + final long startTime = System.currentTimeMillis(); + int commonBlocksFound = 0; + boolean wereNewRequestsMade = false; + + for (Peer peer : peers) { + // Are we shutting down? + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + // Check if we can use the cached common block data, by comparing the peer's current chain tip against the peer's chain tip when we last found our common block + if (peer.canUseCachedCommonBlockData()) { + LOGGER.debug(String.format("Skipping peer %s because we already have the latest common block data in our cache. Cached common block sig is %.08s", peer, Base58.encode(peer.getCommonBlockData().getCommonBlockSummary().getSignature()))); + commonBlocksFound++; + continue; + } + + // Cached data is stale, so clear it and repopulate + peer.setCommonBlockData(null); + + // Search for the common block + Synchronizer.getInstance().findCommonBlockWithPeer(peer, repository); + if (peer.getCommonBlockData() != null) + commonBlocksFound++; + + // This round wasn't served entirely from the cache, so we may want to log the results + wereNewRequestsMade = true; + } + + if (wereNewRequestsMade) { + final long totalTimeTaken = System.currentTimeMillis() - startTime; + LOGGER.debug(String.format("Finished searching for common blocks with %d peer%s. Found: %d. Total time taken: %d ms", peers.size(), (peers.size() != 1 ? "s" : ""), commonBlocksFound, totalTimeTaken)); + } + + return SynchronizationResult.OK; + } finally { + repository.discardChanges(); // Free repository locks, if any, also in case anything went wrong + } + } catch (DataException e) { + LOGGER.error("Repository issue during synchronization with peer", e); + return SynchronizationResult.REPOSITORY_ISSUE; + } + } + + /** + * Attempt to find the find our common block with supplied peer. + * If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later. + *

+ * Will return SynchronizationResult.OK on success. + *

+ * @param peer + * @param repository + * @return SynchronizationResult.OK if the process completed successfully, or a different SynchronizationResult if something went wrong. + * @throws InterruptedException + */ + public SynchronizationResult findCommonBlockWithPeer(Peer peer, Repository repository) throws InterruptedException { + try { + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + final int ourInitialHeight = ourLatestBlockData.getHeight(); + + PeerChainTipData peerChainTipData = peer.getChainTipData(); + int peerHeight = peerChainTipData.getLastHeight(); + byte[] peersLastBlockSignature = peerChainTipData.getLastBlockSignature(); + + byte[] ourLastBlockSignature = ourLatestBlockData.getSignature(); + LOGGER.debug(String.format("Fetching summaries from peer %s at height %d, sig %.8s, ts %d; our height %d, sig %.8s, ts %d", peer, + peerHeight, Base58.encode(peersLastBlockSignature), peer.getChainTipData().getLastBlockTimestamp(), + ourInitialHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp())); + + List peerBlockSummaries = new ArrayList<>(); + SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, false, peerBlockSummaries, false); + if (findCommonBlockResult != SynchronizationResult.OK) { + // Logging performed by fetchSummariesFromCommonBlock() above + peer.setCommonBlockData(null); + return findCommonBlockResult; + } + + // First summary is common block + final BlockData commonBlockData = repository.getBlockRepository().fromSignature(peerBlockSummaries.get(0).getSignature()); + final BlockSummaryData commonBlockSummary = new BlockSummaryData(commonBlockData); + final int commonBlockHeight = commonBlockData.getHeight(); + final byte[] commonBlockSig = commonBlockData.getSignature(); + final String commonBlockSig58 = Base58.encode(commonBlockSig); + LOGGER.debug(String.format("Common block with peer %s is at height %d, sig %.8s, ts %d", peer, + commonBlockHeight, commonBlockSig58, commonBlockData.getTimestamp())); + peerBlockSummaries.remove(0); + + // Store the common block summary against the peer, and the current chain tip (for caching) + peer.setCommonBlockData(new CommonBlockData(commonBlockSummary, peerChainTipData)); + + return SynchronizationResult.OK; + } catch (DataException e) { + LOGGER.error("Repository issue during synchronization with peer", e); + return SynchronizationResult.REPOSITORY_ISSUE; + } + } + + + /** + * Compare a list of peers to determine the best peer(s) to sync to next. + *

+ * Will return a filtered list of peers on success, or an identical list of peers on failure. + * This allows us to fall back to legacy behaviour (random selection from the entire list of peers), if we are unable to make the comparison. + *

+ * @param peers + * @return a list of peers, possibly filtered. + * @throws InterruptedException + */ + public List comparePeers(List peers) throws InterruptedException { + try (final Repository repository = RepositoryManager.getRepository()) { + try { + + // If our latest block is very old, it's best that we don't try and determine the best peers to sync to. + // This is because it can involve very large chain comparisons, which is too intensive. + // In reality, most forking problems occur near the chain tips, so we will reserve this functionality for those situations. + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null) + return peers; + + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + LOGGER.debug(String.format("Our latest block is very old, so we won't filter the peers list")); + return peers; + } + + // We will switch to a new chain weight consensus algorithm at a hard fork, so determine if this has happened yet + boolean usingSameLengthChainWeight = (NTP.getTime() >= BlockChain.getInstance().getCalcChainWeightTimestamp()); + LOGGER.debug(String.format("Using %s chain weight consensus algorithm", (usingSameLengthChainWeight ? "same-length" : "variable-length"))); + + // Retrieve a list of unique common blocks from this list of peers + List commonBlocks = this.uniqueCommonBlocks(peers); + + // Order common blocks by height, in ascending order + // This is essential for the logic below to make the correct decisions when discarding chains - do not remove + commonBlocks.sort((b1, b2) -> Integer.valueOf(b1.getHeight()).compareTo(Integer.valueOf(b2.getHeight()))); + + // Get our latest height + final int ourHeight = ourLatestBlockData.getHeight(); + + // Create a placeholder to track of common blocks that we can discard due to being inferior chains + int dropPeersAfterCommonBlockHeight = 0; + + NumberFormat accurateFormatter = new DecimalFormat("0.################E0"); + + // Remove peers with no common block data + Iterator iterator = peers.iterator(); + while (iterator.hasNext()) { + Peer peer = (Peer) iterator.next(); + if (peer.getCommonBlockData() == null) { + LOGGER.debug(String.format("Removed peer %s because it has no common block data", peer)); + iterator.remove(); + } + } + + // Loop through each group of common blocks + for (BlockSummaryData commonBlockSummary : commonBlocks) { + List peersSharingCommonBlock = peers.stream().filter(peer -> peer.getCommonBlockData().getCommonBlockSummary().equals(commonBlockSummary)).collect(Collectors.toList()); + + // Check if we need to discard this group of peers + if (dropPeersAfterCommonBlockHeight > 0) { + if (commonBlockSummary.getHeight() > dropPeersAfterCommonBlockHeight) { + // We have already determined that the correct chain diverged from a lower height. We are safe to skip these peers. + for (Peer peer : peersSharingCommonBlock) { + LOGGER.debug(String.format("Peer %s has common block at height %d but the superior chain is at height %d. Removing it from this round.", peer, commonBlockSummary.getHeight(), dropPeersAfterCommonBlockHeight)); + this.addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature()); + } + continue; + } + } + + // Calculate the length of the shortest peer chain sharing this common block, including our chain + final int ourAdditionalBlocksAfterCommonBlock = ourHeight - commonBlockSummary.getHeight(); + int minChainLength = this.calculateMinChainLengthOfPeers(peersSharingCommonBlock, commonBlockSummary); + + // Fetch block summaries from each peer + for (Peer peer : peersSharingCommonBlock) { + + // If we're shutting down, just return the latest peer list + if (Controller.isStopping()) + return peers; + + // Count the number of blocks this peer has beyond our common block + final PeerChainTipData peerChainTipData = peer.getChainTipData(); + final int peerHeight = peerChainTipData.getLastHeight(); + final byte[] peerLastBlockSignature = peerChainTipData.getLastBlockSignature(); + final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); + // Limit the number of blocks we are comparing. FUTURE: we could request more in batches, but there may not be a case when this is needed + int summariesRequired = Math.min(peerAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE); + + // Check if we can use the cached common block summaries, by comparing the peer's current chain tip against the peer's chain tip when we last found our common block + boolean useCachedSummaries = false; + if (peer.canUseCachedCommonBlockData()) { + if (peer.getCommonBlockData().getBlockSummariesAfterCommonBlock() != null) { + if (peer.getCommonBlockData().getBlockSummariesAfterCommonBlock().size() == summariesRequired) { + LOGGER.trace(String.format("Using cached block summaries for peer %s", peer)); + useCachedSummaries = true; + } + } + } + + if (useCachedSummaries == false) { + if (summariesRequired > 0) { + LOGGER.trace(String.format("Requesting %d block summar%s from peer %s after common block %.8s. Peer height: %d", summariesRequired, (summariesRequired != 1 ? "ies" : "y"), peer, Base58.encode(commonBlockSummary.getSignature()), peerHeight)); + + // Forget any cached summaries + peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(null); + + // Request new block summaries + List blockSummaries = this.getBlockSummaries(peer, commonBlockSummary.getSignature(), summariesRequired); + if (blockSummaries != null) { + LOGGER.trace(String.format("Peer %s returned %d block summar%s", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"))); + + if (blockSummaries.size() < summariesRequired) + // This could mean that the peer has re-orged. Exclude this peer until they return the summaries we expect. + LOGGER.debug(String.format("Peer %s returned %d block summar%s instead of expected %d - excluding them from this round", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"), summariesRequired)); + else if (blockSummaryWithSignature(peerLastBlockSignature, blockSummaries) == null) + // We don't have a block summary for the peer's reported chain tip, so should exclude it + LOGGER.debug(String.format("Peer %s didn't return a block summary with signature %.8s - excluding them from this round", peer, Base58.encode(peerLastBlockSignature))); + else + // All looks good, so store the retrieved block summaries in the peer's cache + peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(blockSummaries); + } + } else { + // There are no block summaries after this common block + peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(null); + } + } + + // Ignore this peer if it holds an invalid block + if (this.containsInvalidBlockSummary(peer.getCommonBlockData().getBlockSummariesAfterCommonBlock())) { + LOGGER.debug("Ignoring peer %s because it holds an invalid block", peer); + peers.remove(peer); + } + + // Reduce minChainLength if needed. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength + List peerBlockSummaries = peer.getCommonBlockData().getBlockSummariesAfterCommonBlock(); + if (peerBlockSummaries != null && peerBlockSummaries.size() > 0) + if (peerBlockSummaries.size() < minChainLength) + minChainLength = peerBlockSummaries.size(); + } + + // Fetch our corresponding block summaries. Limit to MAXIMUM_REQUEST_SIZE, in order to make the comparison fairer, as peers have been limited too + final int ourSummariesRequired = Math.min(ourAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE); + LOGGER.trace(String.format("About to fetch our block summaries from %d to %d. Our height: %d", commonBlockSummary.getHeight() + 1, commonBlockSummary.getHeight() + ourSummariesRequired, ourHeight)); + List ourBlockSummaries = repository.getBlockRepository().getBlockSummaries(commonBlockSummary.getHeight() + 1, commonBlockSummary.getHeight() + ourSummariesRequired); + if (ourBlockSummaries.isEmpty()) { + LOGGER.debug(String.format("We don't have any block summaries so can't compare our chain against peers with this common block. We can still compare them against each other.")); + } + else { + populateBlockSummariesMinterLevels(repository, ourBlockSummaries); + // Reduce minChainLength if we have less summaries + if (ourBlockSummaries.size() < minChainLength) + minChainLength = ourBlockSummaries.size(); + } + + // Create array to hold peers for comparison + List superiorPeersForComparison = new ArrayList<>(); + + // Calculate max height for chain weight comparisons + int maxHeightForChainWeightComparisons = commonBlockSummary.getHeight() + minChainLength; + + // Calculate our chain weight + BigInteger ourChainWeight = BigInteger.valueOf(0); + if (ourBlockSummaries.size() > 0) + ourChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), ourBlockSummaries, maxHeightForChainWeightComparisons); + + LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", (usingSameLengthChainWeight ? minChainLength : ourBlockSummaries.size()), accurateFormatter.format(ourChainWeight))); + + LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature()))); + for (Peer peer : peersSharingCommonBlock) { + final int peerHeight = peer.getChainTipData().getLastHeight(); + final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); + final CommonBlockData peerCommonBlockData = peer.getCommonBlockData(); + + if (peerCommonBlockData == null || peerCommonBlockData.getBlockSummariesAfterCommonBlock() == null || peerCommonBlockData.getBlockSummariesAfterCommonBlock().isEmpty()) { + // No response - remove this peer for now + LOGGER.debug(String.format("Peer %s doesn't have any block summaries - removing it from this round", peer)); + peers.remove(peer); + continue; + } + + final List peerBlockSummariesAfterCommonBlock = peerCommonBlockData.getBlockSummariesAfterCommonBlock(); + populateBlockSummariesMinterLevels(repository, peerBlockSummariesAfterCommonBlock); + + // Calculate cumulative chain weight of this blockchain subset, from common block to highest mutual block held by all peers in this group. + LOGGER.debug(String.format("About to calculate chain weight based on %d blocks for peer %s with common block %.8s (peer has %d blocks after common block)", (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peer, Base58.encode(commonBlockSummary.getSignature()), peerAdditionalBlocksAfterCommonBlock)); + BigInteger peerChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), peerBlockSummariesAfterCommonBlock, maxHeightForChainWeightComparisons); + peer.getCommonBlockData().setChainWeight(peerChainWeight); + LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), accurateFormatter.format(peerChainWeight))); + + // Compare against our chain - if our blockchain has greater weight then don't synchronize with peer (or any others in this group) + if (ourChainWeight.compareTo(peerChainWeight) > 0) { + // This peer is on an inferior chain - remove it + LOGGER.debug(String.format("Peer %s is on an inferior chain to us - removing it from this round", peer)); + peers.remove(peer); + } + else { + // Our chain is inferior or equal + LOGGER.debug(String.format("Peer %s is on an equal or better chain to us. We will compare the other peers sharing this common block against each other, and drop all peers sharing higher common blocks.", peer)); + dropPeersAfterCommonBlockHeight = commonBlockSummary.getHeight(); + superiorPeersForComparison.add(peer); + } + } + + // Now that we have selected the best peers, compare them against each other and remove any with lower weights + if (superiorPeersForComparison.size() > 0) { + BigInteger bestChainWeight = null; + for (Peer peer : superiorPeersForComparison) { + // Increase bestChainWeight if needed + if (bestChainWeight == null || peer.getCommonBlockData().getChainWeight().compareTo(bestChainWeight) >= 0) + bestChainWeight = peer.getCommonBlockData().getChainWeight(); + } + for (Peer peer : superiorPeersForComparison) { + // Check if we should discard an inferior peer + if (peer.getCommonBlockData().getChainWeight().compareTo(bestChainWeight) < 0) { + BigInteger difference = bestChainWeight.subtract(peer.getCommonBlockData().getChainWeight()); + LOGGER.debug(String.format("Peer %s has a lower chain weight (difference: %s) than other peer(s) in this group - removing it from this round.", peer, accurateFormatter.format(difference))); + peers.remove(peer); + } + } + // FUTURE: we may want to prefer peers with additional blocks, and compare the additional blocks against each other. + // This would fast track us to the best candidate for the latest block. + // Right now, peers with the exact same chain as us are treated equally to those with an additional block. + } + } + + return peers; + } finally { + repository.discardChanges(); // Free repository locks, if any, also in case anything went wrong + } + } catch (DataException e) { + LOGGER.error("Repository issue during peer comparison", e); + return peers; + } + } + + private List uniqueCommonBlocks(List peers) { + List commonBlocks = new ArrayList<>(); + + for (Peer peer : peers) { + if (peer.getCommonBlockData() != null && peer.getCommonBlockData().getCommonBlockSummary() != null) { + LOGGER.trace(String.format("Peer %s has common block %.8s", peer, Base58.encode(peer.getCommonBlockData().getCommonBlockSummary().getSignature()))); + + BlockSummaryData commonBlockSummary = peer.getCommonBlockData().getCommonBlockSummary(); + if (!commonBlocks.contains(commonBlockSummary)) + commonBlocks.add(commonBlockSummary); + } + else { + LOGGER.trace(String.format("Peer %s has no common block data. Skipping...", peer)); + } + } + + return commonBlocks; + } + + private int calculateMinChainLengthOfPeers(List peersSharingCommonBlock, BlockSummaryData commonBlockSummary) { + // Calculate the length of the shortest peer chain sharing this common block + int minChainLength = 0; + for (Peer peer : peersSharingCommonBlock) { + final int peerHeight = peer.getChainTipData().getLastHeight(); + final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); + + if (peerAdditionalBlocksAfterCommonBlock < minChainLength || minChainLength == 0) + minChainLength = peerAdditionalBlocksAfterCommonBlock; + } + return minChainLength; + } + + private BlockSummaryData blockSummaryWithSignature(byte[] signature, List blockSummaries) { + if (blockSummaries != null) + return blockSummaries.stream().filter(blockSummary -> Arrays.equals(blockSummary.getSignature(), signature)).findAny().orElse(null); + return null; + } + + + + /* Invalid block signature tracking */ + + private void addInvalidBlockSignature(byte[] signature) { + Long now = NTP.getTime(); + if (now == null) { + return; + } + + // Add or update existing entry + String sig58 = Base58.encode(signature); + invalidBlockSignatures.put(sig58, now); + } + private void deleteOlderInvalidSignatures(Long now) { + if (now == null) { + return; + } + + // Delete signatures with older timestamps + Iterator it = invalidBlockSignatures.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry pair = (Map.Entry)it.next(); + Long lastSeen = (Long) pair.getValue(); + + // Remove signature if we haven't seen it for more than 1 hour + if (now - lastSeen > 60 * 60 * 1000L) { + it.remove(); + } + } + } + private boolean containsInvalidBlockSummary(List blockSummaries) { + if (blockSummaries == null || invalidBlockSignatures == null) { + return false; + } + + // Loop through our known invalid blocks and check each one against supplied block summaries + for (String invalidSignature58 : invalidBlockSignatures.keySet()) { + byte[] invalidSignature = Base58.decode(invalidSignature58); + for (BlockSummaryData blockSummary : blockSummaries) { + byte[] signature = blockSummary.getSignature(); + if (Arrays.equals(signature, invalidSignature)) { + return true; + } + } + } + return false; + } + private boolean containsInvalidBlockSignature(List blockSignatures) { + if (blockSignatures == null || invalidBlockSignatures == null) { + return false; + } + + // Loop through our known invalid blocks and check each one against supplied block signatures + for (String invalidSignature58 : invalidBlockSignatures.keySet()) { + byte[] invalidSignature = Base58.decode(invalidSignature58); + for (byte[] signature : blockSignatures) { + if (Arrays.equals(signature, invalidSignature)) { + return true; + } + } + } + return false; + } + + /** * Attempt to synchronize blockchain with peer. *

@@ -73,9 +872,11 @@ public SynchronizationResult synchronize(Peer peer, boolean force) throws Interr // Make sure we're the only thread modifying the blockchain // If we're already synchronizing with another peer then this will also return fast ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - if (!blockchainLock.tryLock()) + if (!blockchainLock.tryLock(3, TimeUnit.SECONDS)) { // Wasn't peer's fault we couldn't sync + LOGGER.info("Synchronizer couldn't acquire blockchain lock"); return SynchronizationResult.NO_BLOCKCHAIN_LOCK; + } try { try (final Repository repository = RepositoryManager.getRepository()) { @@ -88,15 +889,31 @@ public SynchronizationResult synchronize(Peer peer, boolean force) throws Interr byte[] peersLastBlockSignature = peerChainTipData.getLastBlockSignature(); byte[] ourLastBlockSignature = ourLatestBlockData.getSignature(); - LOGGER.debug(String.format("Synchronizing with peer %s at height %d, sig %.8s, ts %d; our height %d, sig %.8s, ts %d", peer, + String syncString = String.format("Synchronizing with peer %s at height %d, sig %.8s, ts %d; our height %d, sig %.8s, ts %d", peer, peerHeight, Base58.encode(peersLastBlockSignature), peer.getChainTipData().getLastBlockTimestamp(), - ourInitialHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp())); + ourInitialHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp()); + LOGGER.info(syncString); + + // Reset last re-org size as we are starting a new sync round + this.lastReorgSize = 0; + + // Set the initial value of timeValidBlockLastReceived if it's null + Long now = NTP.getTime(); + if (this.timeValidBlockLastReceived == null) { + this.timeValidBlockLastReceived = now; + } + + // Delete invalid signatures with older timestamps + this.deleteOlderInvalidSignatures(now); List peerBlockSummaries = new ArrayList<>(); - SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, force, peerBlockSummaries); - if (findCommonBlockResult != SynchronizationResult.OK) + SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, force, peerBlockSummaries, true); + if (findCommonBlockResult != SynchronizationResult.OK) { // Logging performed by fetchSummariesFromCommonBlock() above + // Clear our common block cache for this peer + peer.setCommonBlockData(null); return findCommonBlockResult; + } // First summary is common block final BlockData commonBlockData = repository.getBlockRepository().fromSignature(peerBlockSummaries.get(0).getSignature()); @@ -126,175 +943,39 @@ public SynchronizationResult synchronize(Peer peer, boolean force) throws Interr // Unless we're doing a forced sync, we might need to compare blocks after common block if (!force && ourInitialHeight > commonBlockHeight) { - // If our latest block is very old, we're very behind and should ditch our fork. - final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); - if (minLatestBlockTimestamp == null) - return SynchronizationResult.REPOSITORY_ISSUE; - - if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { - LOGGER.info(String.format("Ditching our chain after height %d as our latest block is very old", commonBlockHeight)); - } else { - // Compare chain weights - - LOGGER.debug(String.format("Comparing chains from block %d with peer %s", commonBlockHeight + 1, peer)); - - // Fetch remaining peer's block summaries (which we also use to fill signatures list) - int peerBlockCount = peerHeight - commonBlockHeight; - - while (peerBlockSummaries.size() < peerBlockCount) { - if (Controller.isStopping()) - return SynchronizationResult.SHUTTING_DOWN; - - int lastSummaryHeight = commonBlockHeight + peerBlockSummaries.size(); - byte[] previousSignature; - if (peerBlockSummaries.isEmpty()) - previousSignature = commonBlockSig; - else - previousSignature = peerBlockSummaries.get(peerBlockSummaries.size() - 1).getSignature(); - - List moreBlockSummaries = this.getBlockSummaries(peer, previousSignature, peerBlockCount - peerBlockSummaries.size()); - - if (moreBlockSummaries == null || moreBlockSummaries.isEmpty()) { - LOGGER.info(String.format("Peer %s failed to respond with block summaries after height %d, sig %.8s", peer, - lastSummaryHeight, Base58.encode(previousSignature))); - return SynchronizationResult.NO_REPLY; - } - - // Check peer sent valid heights - for (int i = 0; i < moreBlockSummaries.size(); ++i) { - ++lastSummaryHeight; - - BlockSummaryData blockSummary = moreBlockSummaries.get(i); - - if (blockSummary.getHeight() != lastSummaryHeight) { - LOGGER.info(String.format("Peer %s responded with invalid block summary for height %d, sig %.8s", peer, - lastSummaryHeight, Base58.encode(blockSummary.getSignature()))); - return SynchronizationResult.NO_REPLY; - } - } - - peerBlockSummaries.addAll(moreBlockSummaries); - } - - // Fetch our corresponding block summaries - List ourBlockSummaries = repository.getBlockRepository().getBlockSummaries(commonBlockHeight + 1, ourInitialHeight); - - // Populate minter account levels for both lists of block summaries - populateBlockSummariesMinterLevels(repository, peerBlockSummaries); - populateBlockSummariesMinterLevels(repository, ourBlockSummaries); - - // Calculate cumulative chain weights of both blockchain subsets, from common block to highest mutual block. - BigInteger ourChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries); - BigInteger peerChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, peerBlockSummaries); - - // If our blockchain has greater weight then don't synchronize with peer - if (ourChainWeight.compareTo(peerChainWeight) >= 0) { - LOGGER.debug(String.format("Not synchronizing with peer %s as we have better blockchain", peer)); - NumberFormat formatter = new DecimalFormat("0.###E0"); - LOGGER.debug(String.format("Our chain weight: %s, peer's chain weight: %s (higher is better)", formatter.format(ourChainWeight), formatter.format(peerChainWeight))); - return SynchronizationResult.INFERIOR_CHAIN; - } - } + SynchronizationResult chainCompareResult = compareChains(repository, commonBlockData, ourLatestBlockData, peer, peerHeight, peerBlockSummaries); + if (chainCompareResult != SynchronizationResult.OK) + return chainCompareResult; } - int ourHeight = ourInitialHeight; - if (ourHeight > commonBlockHeight) { - // Unwind to common block (unless common block is our latest block) - LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s", commonBlockHeight, commonBlockSig58)); - - while (ourHeight > commonBlockHeight) { - if (Controller.isStopping()) - return SynchronizationResult.SHUTTING_DOWN; - - BlockData blockData = repository.getBlockRepository().fromHeight(ourHeight); - Block block = new Block(repository, blockData); - block.orphan(); - - --ourHeight; - } - - LOGGER.debug(String.format("Orphaned blocks back to height %d, sig %.8s - fetching blocks from peer %s", commonBlockHeight, commonBlockSig58, peer)); + SynchronizationResult syncResult = null; + if (commonBlockHeight < ourInitialHeight) { + // Peer's chain is better, sync to that one + syncResult = syncToPeerChain(repository, commonBlockData, ourInitialHeight, peer, peerHeight, peerBlockSummaries); } else { - LOGGER.debug(String.format("Fetching new blocks from peer %s", peer)); + // Simply fetch and apply blocks as they arrive + syncResult = applyNewBlocks(repository, commonBlockData, ourInitialHeight, peer, peerHeight, peerBlockSummaries); } - // Fetch, and apply, blocks from peer - byte[] latestPeerSignature = commonBlockSig; - int maxBatchHeight = commonBlockHeight + SYNC_BATCH_SIZE; - - // Convert any block summaries from above into signatures to request from peer - List peerBlockSignatures = peerBlockSummaries.stream().map(BlockSummaryData::getSignature).collect(Collectors.toList()); - - while (ourHeight < peerHeight && ourHeight < maxBatchHeight) { - if (Controller.isStopping()) - return SynchronizationResult.SHUTTING_DOWN; - - // Do we need more signatures? - if (peerBlockSignatures.isEmpty()) { - int numberRequested = maxBatchHeight - ourHeight; - LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s", - numberRequested, (numberRequested != 1 ? "s": ""), ourHeight, Base58.encode(latestPeerSignature))); - - peerBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberRequested); - - if (peerBlockSignatures == null || peerBlockSignatures.isEmpty()) { - LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, - ourHeight, Base58.encode(latestPeerSignature))); - return SynchronizationResult.NO_REPLY; - } - - LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : ""))); - } - - latestPeerSignature = peerBlockSignatures.get(0); - peerBlockSignatures.remove(0); - ++ourHeight; - - Block newBlock = this.fetchBlock(repository, peer, latestPeerSignature); - - if (newBlock == null) { - LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, - ourHeight, Base58.encode(latestPeerSignature))); - return SynchronizationResult.NO_REPLY; - } - - if (!newBlock.isSignatureValid()) { - LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer, - ourHeight, Base58.encode(latestPeerSignature))); - return SynchronizationResult.INVALID_DATA; - } - - // Transactions are transmitted without approval status so determine that now - for (Transaction transaction : newBlock.getTransactions()) - transaction.setInitialApprovalStatus(); - - ValidationResult blockResult = newBlock.isValid(); - if (blockResult != ValidationResult.OK) { - LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer, - ourHeight, Base58.encode(latestPeerSignature), blockResult.name())); - return SynchronizationResult.INVALID_DATA; - } - - // Save transactions attached to this block - for (Transaction transaction : newBlock.getTransactions()) { - TransactionData transactionData = transaction.getTransactionData(); - repository.getTransactionRepository().save(transactionData); - } - - newBlock.process(); - - // If we've grown our blockchain then at least save progress so far - if (ourHeight > ourInitialHeight) - repository.saveChanges(); - } + if (syncResult != SynchronizationResult.OK) + return syncResult; // Commit repository.saveChanges(); + // Create string for logging final BlockData newLatestBlockData = repository.getBlockRepository().getLastBlock(); - LOGGER.info(String.format("Synchronized with peer %s to height %d, sig %.8s, ts: %d", peer, + String syncLog = String.format("Synchronized with peer %s to height %d, sig %.8s, ts: %d", peer, newLatestBlockData.getHeight(), Base58.encode(newLatestBlockData.getSignature()), - newLatestBlockData.getTimestamp())); + newLatestBlockData.getTimestamp()); + + // Append re-org info + if (this.lastReorgSize > 0) { + syncLog = syncLog.concat(String.format(", size: %d", this.lastReorgSize)); + } + + // Log sync info + LOGGER.info(syncLog); return SynchronizationResult.OK; } finally { @@ -317,7 +998,7 @@ public SynchronizationResult synchronize(Peer peer, boolean force) throws Interr * @throws DataException * @throws InterruptedException */ - private SynchronizationResult fetchSummariesFromCommonBlock(Repository repository, Peer peer, int ourHeight, boolean force, List blockSummariesFromCommon) throws DataException, InterruptedException { + public SynchronizationResult fetchSummariesFromCommonBlock(Repository repository, Peer peer, int ourHeight, boolean force, List blockSummariesFromCommon, boolean infoLogWhenNotFound) throws DataException, InterruptedException { // Start by asking for a few recent block hashes as this will cover a majority of reorgs // Failing that, back off exponentially int step = INITIAL_BLOCK_STEP; @@ -346,8 +1027,12 @@ private SynchronizationResult fetchSummariesFromCommonBlock(Repository repositor blockSummariesBatch = this.getBlockSummaries(peer, testSignature, step); if (blockSummariesBatch == null) { + if (infoLogWhenNotFound) + LOGGER.info(String.format("Error while trying to find common block with peer %s", peer)); + else + LOGGER.debug(String.format("Error while trying to find common block with peer %s", peer)); + // No response - give up this time - LOGGER.info(String.format("Error while trying to find common block with peer %s", peer)); return SynchronizationResult.NO_REPLY; } @@ -383,14 +1068,423 @@ private SynchronizationResult fetchSummariesFromCommonBlock(Repository repositor blockSummariesFromCommon.addAll(blockSummariesBatch); // Trim summaries so that first summary is common block. - // Currently we work back from the end until we hit a block we also have. + // Currently we work forward from common block until we hit a block we don't have // TODO: rewrite as modified binary search! - for (int i = blockSummariesFromCommon.size() - 1; i > 0; --i) { - if (repository.getBlockRepository().exists(blockSummariesFromCommon.get(i).getSignature())) { - // Note: index i isn't cleared: List.subList is fromIndex inclusive to toIndex exclusive - blockSummariesFromCommon.subList(0, i).clear(); + int i; + for (i = 1; i < blockSummariesFromCommon.size(); ++i) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + if (!repository.getBlockRepository().exists(blockSummariesFromCommon.get(i).getSignature())) + break; + } + + // Note: index i - 1 isn't cleared: List.subList is fromIndex inclusive to toIndex exclusive + blockSummariesFromCommon.subList(0, i - 1).clear(); + + return SynchronizationResult.OK; + } + + private SynchronizationResult compareChains(Repository repository, BlockData commonBlockData, BlockData ourLatestBlockData, + Peer peer, int peerHeight, List peerBlockSummaries) throws DataException, InterruptedException { + final int commonBlockHeight = commonBlockData.getHeight(); + final byte[] commonBlockSig = commonBlockData.getSignature(); + + // If our latest block is very old, we're very behind and should ditch our fork. + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null) + return SynchronizationResult.REPOSITORY_ISSUE; + + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + LOGGER.info(String.format("Ditching our chain after height %d", commonBlockHeight)); + } else { + // Compare chain weights + + LOGGER.debug(String.format("Comparing chains from block %d with peer %s", commonBlockHeight + 1, peer)); + + // Fetch remaining peer's block summaries (which we also use to fill signatures list) + int peerBlockCount = peerHeight - commonBlockHeight; + + while (peerBlockSummaries.size() < peerBlockCount) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + int lastSummaryHeight = commonBlockHeight + peerBlockSummaries.size(); + byte[] previousSignature; + if (peerBlockSummaries.isEmpty()) + previousSignature = commonBlockSig; + else + previousSignature = peerBlockSummaries.get(peerBlockSummaries.size() - 1).getSignature(); + + List moreBlockSummaries = this.getBlockSummaries(peer, previousSignature, peerBlockCount - peerBlockSummaries.size()); + + if (moreBlockSummaries == null || moreBlockSummaries.isEmpty()) { + LOGGER.info(String.format("Peer %s failed to respond with block summaries after height %d, sig %.8s", peer, + lastSummaryHeight, Base58.encode(previousSignature))); + return SynchronizationResult.NO_REPLY; + } + + // Check peer sent valid heights + for (int i = 0; i < moreBlockSummaries.size(); ++i) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + ++lastSummaryHeight; + + BlockSummaryData blockSummary = moreBlockSummaries.get(i); + + if (blockSummary.getHeight() != lastSummaryHeight) { + LOGGER.info(String.format("Peer %s responded with invalid block summary for height %d, sig %.8s", peer, + lastSummaryHeight, Base58.encode(blockSummary.getSignature()))); + return SynchronizationResult.NO_REPLY; + } + } + + peerBlockSummaries.addAll(moreBlockSummaries); + } + + // Fetch our corresponding block summaries + List ourBlockSummaries = repository.getBlockRepository().getBlockSummaries(commonBlockHeight + 1, ourLatestBlockData.getHeight()); + + // Populate minter account levels for both lists of block summaries + populateBlockSummariesMinterLevels(repository, ourBlockSummaries); + populateBlockSummariesMinterLevels(repository, peerBlockSummaries); + + final int mutualHeight = commonBlockHeight + Math.min(ourBlockSummaries.size(), peerBlockSummaries.size()); + + // Calculate cumulative chain weights of both blockchain subsets, from common block to highest mutual block. + BigInteger ourChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight); + BigInteger peerChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, peerBlockSummaries, mutualHeight); + + NumberFormat accurateFormatter = new DecimalFormat("0.################E0"); + LOGGER.debug(String.format("commonBlockHeight: %d, commonBlockSig: %.8s, ourBlockSummaries.size(): %d, peerBlockSummaries.size(): %d", commonBlockHeight, Base58.encode(commonBlockSig), ourBlockSummaries.size(), peerBlockSummaries.size())); + LOGGER.debug(String.format("Our chain weight: %s, peer's chain weight: %s (higher is better)", accurateFormatter.format(ourChainWeight), accurateFormatter.format(peerChainWeight))); + + // If our blockchain has greater weight then don't synchronize with peer + if (ourChainWeight.compareTo(peerChainWeight) >= 0) { + LOGGER.debug(String.format("Not synchronizing with peer %s as we have better blockchain", peer)); + return SynchronizationResult.INFERIOR_CHAIN; + } + } + + return SynchronizationResult.OK; + } + + private SynchronizationResult syncToPeerChain(Repository repository, BlockData commonBlockData, int ourInitialHeight, + Peer peer, final int peerHeight, List peerBlockSummaries) throws DataException, InterruptedException { + final int commonBlockHeight = commonBlockData.getHeight(); + final byte[] commonBlockSig = commonBlockData.getSignature(); + String commonBlockSig58 = Base58.encode(commonBlockSig); + + byte[] latestPeerSignature = commonBlockSig; + int height = commonBlockHeight; + + LOGGER.debug(() -> String.format("Fetching peer %s chain from height %d, sig %.8s", peer, commonBlockHeight, commonBlockSig58)); + + final int maxRetries = Settings.getInstance().getMaxRetries(); + + // Overall plan: fetch peer's blocks first, then orphan, then apply + + // Convert any leftover (post-common) block summaries into signatures to request from peer + List peerBlockSignatures = peerBlockSummaries.stream().map(BlockSummaryData::getSignature).collect(Collectors.toList()); + + // Keep a list of blocks received so far + List peerBlocks = new ArrayList<>(); + + // Calculate the total number of additional blocks this peer has beyond the common block + int additionalPeerBlocksAfterCommonBlock = peerHeight - commonBlockHeight; + // Subtract the number of signatures that we already have, as we don't need to request them again + int numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size(); + + int retryCount = 0; + while (height < peerHeight) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + // Ensure we don't request more than MAXIMUM_REQUEST_SIZE + int numberRequested = Math.min(numberSignaturesRequired, MAXIMUM_REQUEST_SIZE); + + // Do we need more signatures? + if (peerBlockSignatures.isEmpty() && numberRequested > 0) { + LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s", + numberRequested, (numberRequested != 1 ? "s" : ""), height, Base58.encode(latestPeerSignature))); + + peerBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberRequested); + + if (peerBlockSignatures == null || peerBlockSignatures.isEmpty()) { + LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, + height, Base58.encode(latestPeerSignature))); + + // Clear our cache of common block summaries for this peer, as they are likely to be invalid + CommonBlockData cachedCommonBlockData = peer.getCommonBlockData(); + if (cachedCommonBlockData != null) + cachedCommonBlockData.setBlockSummariesAfterCommonBlock(null); + + // If we have already received newer blocks from this peer that what we have already, go ahead and apply them + if (peerBlocks.size() > 0) { + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1); + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (ourLatestBlockData != null && peerLatestBlock != null && minLatestBlockTimestamp != null) { + + // If our latest block is very old.... + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + // ... and we have received a block that is more recent than our latest block ... + if (peerLatestBlock.getBlockData().getTimestamp() > ourLatestBlockData.getTimestamp()) { + // ... then apply the blocks, as it takes us a step forward. + // This is particularly useful when starting up a node that was on a small fork when it was last shut down. + // In these cases, we now allow the node to sync forward, and get onto the main chain again. + // Without this, we would require that the node syncs ENTIRELY with this peer, + // and any problems downloading a block would cause all progress to be lost. + LOGGER.debug(String.format("Newly received blocks are %d ms newer than our latest block - so we will apply them", peerLatestBlock.getBlockData().getTimestamp() - ourLatestBlockData.getTimestamp())); + break; + } + } + } + } + // Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated or incomplete state + return SynchronizationResult.NO_REPLY; + } + + numberSignaturesRequired = peerHeight - height - peerBlockSignatures.size(); + LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : ""))); + } + + if (peerBlockSignatures.isEmpty()) { + LOGGER.trace(String.format("No more signatures or blocks to request from peer %s", peer)); break; } + + // Catch a block with an invalid signature before orphaning, so that we retain our existing valid candidate + if (this.containsInvalidBlockSignature(peerBlockSignatures)) { + LOGGER.info(String.format("Peer %s sent invalid block signature: %.8s", peer, Base58.encode(latestPeerSignature))); + return SynchronizationResult.INVALID_DATA; + } + + byte[] nextPeerSignature = peerBlockSignatures.get(0); + int nextHeight = height + 1; + + LOGGER.trace(String.format("Fetching block %d, sig %.8s from %s", nextHeight, Base58.encode(nextPeerSignature), peer)); + Block newBlock = this.fetchBlock(repository, peer, nextPeerSignature); + + if (newBlock == null) { + LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, + nextHeight, Base58.encode(nextPeerSignature))); + + if (retryCount >= maxRetries) { + // If we have already received newer blocks from this peer that what we have already, go ahead and apply them + if (peerBlocks.size() > 0) { + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1); + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (ourLatestBlockData != null && peerLatestBlock != null && minLatestBlockTimestamp != null) { + + // If our latest block is very old.... + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + // ... and we have received a block that is more recent than our latest block ... + if (peerLatestBlock.getBlockData().getTimestamp() > ourLatestBlockData.getTimestamp()) { + // ... then apply the blocks, as it takes us a step forward. + // This is particularly useful when starting up a node that was on a small fork when it was last shut down. + // In these cases, we now allow the node to sync forward, and get onto the main chain again. + // Without this, we would require that the node syncs ENTIRELY with this peer, + // and any problems downloading a block would cause all progress to be lost. + LOGGER.debug(String.format("Newly received blocks are %d ms newer than our latest block - so we will apply them", peerLatestBlock.getBlockData().getTimestamp() - ourLatestBlockData.getTimestamp())); + break; + } + } + } + } + // Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated or incomplete state + return SynchronizationResult.NO_REPLY; + + } else { + // Re-fetch signatures, in case the peer is now on a different fork + peerBlockSignatures.clear(); + numberSignaturesRequired = peerHeight - height; + + // Retry until retryCount reaches maxRetries + retryCount++; + int triesRemaining = maxRetries - retryCount; + LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s" : ""))); + continue; + } + } + + // Reset retryCount because the last request succeeded + retryCount = 0; + + LOGGER.trace(String.format("Fetched block %d, sig %.8s from %s", nextHeight, Base58.encode(latestPeerSignature), peer)); + + if (!newBlock.isSignatureValid()) { + LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer, + nextHeight, Base58.encode(latestPeerSignature))); + return SynchronizationResult.INVALID_DATA; + } + + // Transactions are transmitted without approval status so determine that now + for (Transaction transaction : newBlock.getTransactions()) + transaction.setInitialApprovalStatus(); + + peerBlocks.add(newBlock); + + // Now that we've received this block, we can increase our height and move on to the next one + latestPeerSignature = nextPeerSignature; + peerBlockSignatures.remove(0); + ++height; + } + + // Unwind to common block (unless common block is our latest block) + int ourHeight = ourInitialHeight; + LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s. Our height: %d", commonBlockHeight, commonBlockSig58, ourHeight)); + int reorgSize = ourHeight - commonBlockHeight; + + BlockData orphanBlockData = repository.getBlockRepository().fromHeight(ourInitialHeight); + while (ourHeight > commonBlockHeight) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + Block block = new Block(repository, orphanBlockData); + block.orphan(); + + LOGGER.trace(String.format("Orphaned block height %d, sig %.8s", ourHeight, Base58.encode(orphanBlockData.getSignature()))); + + repository.saveChanges(); + + --ourHeight; + orphanBlockData = repository.getBlockRepository().fromHeight(ourHeight); + + repository.discardChanges(); // clear transaction status to prevent deadlocks + Controller.getInstance().onOrphanedBlock(orphanBlockData); + } + + LOGGER.debug(String.format("Orphaned blocks back to height %d, sig %.8s - applying new blocks from peer %s", commonBlockHeight, commonBlockSig58, peer)); + + for (Block newBlock : peerBlocks) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + newBlock.preProcess(); + + ValidationResult blockResult = newBlock.isValid(); + if (blockResult != ValidationResult.OK) { + LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer, + newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getSignature()), blockResult.name())); + this.addInvalidBlockSignature(newBlock.getSignature()); + this.timeInvalidBlockLastReceived = NTP.getTime(); + return SynchronizationResult.INVALID_DATA; + } + + // Block is valid + this.timeValidBlockLastReceived = NTP.getTime(); + + // Save transactions attached to this block + for (Transaction transaction : newBlock.getTransactions()) { + TransactionData transactionData = transaction.getTransactionData(); + repository.getTransactionRepository().save(transactionData); + } + + newBlock.process(); + + LOGGER.trace(String.format("Processed block height %d, sig %.8s", newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature()))); + + repository.saveChanges(); + + Controller.getInstance().onNewBlock(newBlock.getBlockData()); + } + + this.lastReorgSize = reorgSize; + return SynchronizationResult.OK; + } + + private SynchronizationResult applyNewBlocks(Repository repository, BlockData commonBlockData, int ourInitialHeight, + Peer peer, int peerHeight, List peerBlockSummaries) throws InterruptedException, DataException { + LOGGER.debug(String.format("Fetching new blocks from peer %s", peer)); + + final int commonBlockHeight = commonBlockData.getHeight(); + final byte[] commonBlockSig = commonBlockData.getSignature(); + + int ourHeight = ourInitialHeight; + + // Fetch, and apply, blocks from peer + byte[] latestPeerSignature = commonBlockSig; + int maxBatchHeight = commonBlockHeight + SYNC_BATCH_SIZE; + + // Convert any block summaries from above into signatures to request from peer + List peerBlockSignatures = peerBlockSummaries.stream().map(BlockSummaryData::getSignature).collect(Collectors.toList()); + + while (ourHeight < peerHeight && ourHeight < maxBatchHeight) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + // Do we need more signatures? + if (peerBlockSignatures.isEmpty()) { + int numberRequested = Math.min(maxBatchHeight - ourHeight, MAXIMUM_REQUEST_SIZE); + + LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s", + numberRequested, (numberRequested != 1 ? "s": ""), ourHeight, Base58.encode(latestPeerSignature))); + + peerBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberRequested); + + if (peerBlockSignatures == null || peerBlockSignatures.isEmpty()) { + LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, + ourHeight, Base58.encode(latestPeerSignature))); + return SynchronizationResult.NO_REPLY; + } + + LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : ""))); + } + + latestPeerSignature = peerBlockSignatures.get(0); + peerBlockSignatures.remove(0); + ++ourHeight; + + LOGGER.trace(String.format("Fetching block %d, sig %.8s from %s", ourHeight, Base58.encode(latestPeerSignature), peer)); + Block newBlock = this.fetchBlock(repository, peer, latestPeerSignature); + LOGGER.trace(String.format("Fetched block %d, sig %.8s from %s", ourHeight, Base58.encode(latestPeerSignature), peer)); + + if (newBlock == null) { + LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, + ourHeight, Base58.encode(latestPeerSignature))); + return SynchronizationResult.NO_REPLY; + } + + if (!newBlock.isSignatureValid()) { + LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer, + ourHeight, Base58.encode(latestPeerSignature))); + return SynchronizationResult.INVALID_DATA; + } + + // Transactions are transmitted without approval status so determine that now + for (Transaction transaction : newBlock.getTransactions()) + transaction.setInitialApprovalStatus(); + + newBlock.preProcess(); + + ValidationResult blockResult = newBlock.isValid(); + if (blockResult != ValidationResult.OK) { + LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer, + ourHeight, Base58.encode(latestPeerSignature), blockResult.name())); + this.addInvalidBlockSignature(newBlock.getSignature()); + this.timeInvalidBlockLastReceived = NTP.getTime(); + return SynchronizationResult.INVALID_DATA; + } + + // Block is valid + this.timeValidBlockLastReceived = NTP.getTime(); + + // Save transactions attached to this block + for (Transaction transaction : newBlock.getTransactions()) { + TransactionData transactionData = transaction.getTransactionData(); + repository.getTransactionRepository().save(transactionData); + } + + newBlock.process(); + + LOGGER.trace(String.format("Processed block height %d, sig %.8s", newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature()))); + + repository.saveChanges(); + + Controller.getInstance().onNewBlock(newBlock.getBlockData()); } return SynchronizationResult.OK; @@ -432,17 +1526,38 @@ private Block fetchBlock(Repository repository, Peer peer, byte[] signature) thr return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates()); } - private void populateBlockSummariesMinterLevels(Repository repository, List blockSummaries) throws DataException { + public void populateBlockSummariesMinterLevels(Repository repository, List blockSummaries) throws DataException { + final int firstBlockHeight = blockSummaries.get(0).getHeight(); + for (int i = 0; i < blockSummaries.size(); ++i) { + if (Controller.isStopping()) + return; + BlockSummaryData blockSummary = blockSummaries.get(i); // Qortal: minter is always a reward-share, so find actual minter and get their effective minting level int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockSummary.getMinterPublicKey()); if (minterLevel == 0) { - // We don't want to throw, or use zero, as this will kill Controller thread and make client unstable. - // So we log this but use 1 instead - LOGGER.warn(String.format("Unexpected zero effective minter level for reward-share %s - using 1 instead!", Base58.encode(blockSummary.getMinterPublicKey()))); - minterLevel = 1; + // It looks like this block's minter's reward-share has been cancelled. + // So search for REWARD_SHARE transactions since common block to find missing minter info + List transactionSignatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(Transaction.TransactionType.REWARD_SHARE, null, firstBlockHeight, null); + + for (byte[] transactionSignature : transactionSignatures) { + RewardShareTransactionData transactionData = (RewardShareTransactionData) repository.getTransactionRepository().fromSignature(transactionSignature); + + if (transactionData != null && Arrays.equals(transactionData.getRewardSharePublicKey(), blockSummary.getMinterPublicKey())) { + Account rewardShareMinter = new PublicKeyAccount(repository, transactionData.getMinterPublicKey()); + minterLevel = rewardShareMinter.getEffectiveMintingLevel(); + break; + } + } + + if (minterLevel == 0) { + // We don't want to throw, or use zero, as this will kill Controller thread and make client unstable. + // So we log this but use 1 instead + LOGGER.debug(() -> String.format("Unexpected zero effective minter level for reward-share %s - using 1 instead!", Base58.encode(blockSummary.getMinterPublicKey()))); + minterLevel = 1; + } } blockSummary.setMinterLevel(minterLevel); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java new file mode 100644 index 000000000..ebff6913b --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -0,0 +1,203 @@ +package org.qortal.controller.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; +import org.qortal.utils.NTP; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class ArbitraryDataBuildManager extends Thread { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildManager.class); + + private static ArbitraryDataBuildManager instance; + + private volatile boolean isStopping = false; + private boolean buildInProgress = false; + + /** + * Map to keep track of arbitrary transaction resources currently being built (or queued). + */ + public Map arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>()); + + /** + * Map to keep track of failed arbitrary transaction builds. + */ + public Map arbitraryDataFailedBuilds = Collections.synchronizedMap(new HashMap<>()); + + + public ArbitraryDataBuildManager() { + + } + + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data Build Manager"); + + try { + // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread) + // This can be expanded to have multiple threads processing the build queue when needed + int threadCount = 5; + ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(threadCount); + for (int i = 0; i < threadCount; i++) { + arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread()); + } + + while (!isStopping) { + // Nothing to do yet + Thread.sleep(5000); + } + + } catch (InterruptedException e) { + // Fall-through to exit thread... + } + } + + public static ArbitraryDataBuildManager getInstance() { + if (instance == null) + instance = new ArbitraryDataBuildManager(); + + return instance; + } + + public void shutdown() { + isStopping = true; + this.interrupt(); + } + + + public void cleanupQueues(Long now) { + if (now == null) { + return; + } + arbitraryDataBuildQueue.entrySet().removeIf(entry -> entry.getValue().hasReachedBuildTimeout(now)); + arbitraryDataFailedBuilds.entrySet().removeIf(entry -> entry.getValue().hasReachedFailureTimeout(now)); + } + + // Build queue + + public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) { + String key = queueItem.getUniqueKey(); + if (key == null) { + return false; + } + + if (this.arbitraryDataBuildQueue == null) { + return false; + } + + if (NTP.getTime() == null) { + // Can't use queues until we have synced the time + return false; + } + + // Don't add builds that have failed recently + if (this.isInFailedBuildsList(queueItem)) { + return false; + } + + if (this.arbitraryDataBuildQueue.put(key, queueItem) != null) { + // Already in queue + return true; + } + + log(queueItem, String.format("Added %s to build queue", queueItem)); + + // Added to queue + return true; + } + + public boolean isInBuildQueue(ArbitraryDataBuildQueueItem queueItem) { + String key = queueItem.getUniqueKey(); + if (key == null) { + return false; + } + + if (this.arbitraryDataBuildQueue == null) { + return false; + } + + if (this.arbitraryDataBuildQueue.containsKey(key)) { + // Already in queue + return true; + } + + // Not in queue + return false; + } + + + // Failed builds + + public boolean addToFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) { + String key = queueItem.getUniqueKey(); + if (key == null) { + return false; + } + + if (this.arbitraryDataFailedBuilds == null) { + return false; + } + + if (NTP.getTime() == null) { + // Can't use queues until we have synced the time + return false; + } + + if (this.arbitraryDataFailedBuilds.put(key, queueItem) != null) { + // Already in list + return true; + } + + log(queueItem, String.format("Added %s to failed builds list", queueItem)); + + // Added to queue + return true; + } + + public boolean isInFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) { + String key = queueItem.getUniqueKey(); + if (key == null) { + return false; + } + + if (this.arbitraryDataFailedBuilds == null) { + return false; + } + + if (this.arbitraryDataFailedBuilds.containsKey(key)) { + // Already in list + return true; + } + + // Not in list + return false; + } + + + public void setBuildInProgress(boolean buildInProgress) { + this.buildInProgress = buildInProgress; + } + + public boolean getBuildInProgress() { + return this.buildInProgress; + } + + private void log(ArbitraryDataBuildQueueItem queueItem, String message) { + if (queueItem == null) { + return; + } + + if (queueItem.isHighPriority()) { + LOGGER.info(message); + } + else { + LOGGER.debug(message); + } + } +} diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java new file mode 100644 index 000000000..0fb685a3f --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java @@ -0,0 +1,122 @@ +package org.qortal.controller.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.controller.Controller; +import org.qortal.repository.DataException; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.util.Comparator; +import java.util.Map; + + +public class ArbitraryDataBuilderThread implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuilderThread.class); + + public ArbitraryDataBuilderThread() { + + } + + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data Builder Thread"); + ArbitraryDataBuildManager buildManager = ArbitraryDataBuildManager.getInstance(); + + while (!Controller.isStopping()) { + try { + Thread.sleep(100); + + if (buildManager.arbitraryDataBuildQueue == null) { + continue; + } + if (buildManager.arbitraryDataBuildQueue.isEmpty()) { + continue; + } + + Long now = NTP.getTime(); + if (now == null) { + continue; + } + + ArbitraryDataBuildQueueItem queueItem = null; + + // Find resources that are queued for building (sorted by highest priority first) + synchronized (buildManager.arbitraryDataBuildQueue) { + Map.Entry next = buildManager.arbitraryDataBuildQueue + .entrySet().stream() + .filter(e -> e.getValue().isQueued()) + .sorted(Comparator.comparing(item -> item.getValue().getPriority())) + .reduce((first, second) -> second).orElse(null); + + if (next == null) { + continue; + } + + queueItem = next.getValue(); + + if (queueItem == null) { + this.removeFromQueue(queueItem); + continue; + } + + // Ignore builds that have failed recently + if (buildManager.isInFailedBuildsList(queueItem)) { + this.removeFromQueue(queueItem); + continue; + } + + // Set the start timestamp, to prevent other threads from building it at the same time + queueItem.prepareForBuild(); + } + + try { + // Perform the build + log(queueItem, String.format("Building %s... priority: %d", queueItem, queueItem.getPriority())); + queueItem.build(); + this.removeFromQueue(queueItem); + log(queueItem, String.format("Finished building %s", queueItem)); + + } catch (MissingDataException e) { + log(queueItem, String.format("Missing data for %s: %s", queueItem, e.getMessage())); + queueItem.setFailed(true); + this.removeFromQueue(queueItem); + // Don't add to the failed builds list, as we may want to retry sooner + + } catch (IOException | DataException | RuntimeException e) { + log(queueItem, String.format("Error building %s: %s", queueItem, e.getMessage())); + // Something went wrong - so remove it from the queue, and add to failed builds list + queueItem.setFailed(true); + buildManager.addToFailedBuildsList(queueItem); + this.removeFromQueue(queueItem); + } + + } catch (InterruptedException e) { + // Time to exit + } + } + } + + private void removeFromQueue(ArbitraryDataBuildQueueItem queueItem) { + if (queueItem == null || queueItem.getUniqueKey() == null) { + return; + } + ArbitraryDataBuildManager.getInstance().arbitraryDataBuildQueue.remove(queueItem.getUniqueKey()); + } + + private void log(ArbitraryDataBuildQueueItem queueItem, String message) { + if (queueItem == null) { + return; + } + + if (queueItem.isHighPriority()) { + LOGGER.info(message); + } + else { + LOGGER.debug(message); + } + } +} diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java new file mode 100644 index 000000000..64916df54 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -0,0 +1,576 @@ +package org.qortal.controller.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.transaction.Transaction; +import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.utils.ArbitraryTransactionUtils; +import org.qortal.utils.Base58; +import org.qortal.utils.FilesystemUtils; +import org.qortal.utils.NTP; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.SecureRandom; +import java.util.*; + +import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD; + +public class ArbitraryDataCleanupManager extends Thread { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCleanupManager.class); + private static final List ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY); + + private static ArbitraryDataCleanupManager instance; + + private volatile boolean isStopping = false; + + /** + * The amount of time that must pass before a file is treated as stale / not recent. + * We can safely delete files created/accessed longer ago that this, if we have a means of + * rebuilding them. The main purpose of this is to avoid deleting files that are currently + * being used by other parts of the system. + */ + private static final long STALE_FILE_TIMEOUT = 60*60*1000L; // 1 hour + + /** + * The number of chunks to delete in a batch when over the capacity limit. + * Storage limits are re-checked after each batch, and there could be a significant + * delay between the processing of each batch as it only occurs after a complete + * cleanup cycle (to allow unwanted chunks to be deleted first). + */ + private static final int CHUNK_DELETION_BATCH_SIZE = 10; + + + /* + TODO: + - Delete files from the _misc folder once they reach a certain age + */ + + + private ArbitraryDataCleanupManager() { + } + + public static ArbitraryDataCleanupManager getInstance() { + if (instance == null) + instance = new ArbitraryDataCleanupManager(); + + return instance; + } + + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data Cleanup Manager"); + + // Paginate queries when fetching arbitrary transactions + final int limit = 100; + int offset = 0; + + try { + while (!isStopping) { + Thread.sleep(30000); + + // Don't run if QDN is disabled + if (!Settings.getInstance().isQdnEnabled()) { + Thread.sleep(60 * 60 * 1000L); + continue; + } + + Long now = NTP.getTime(); + if (now == null) { + // Don't attempt to make decisions if we haven't synced our time yet + continue; + } + + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + + // Wait until storage capacity has been calculated + if (!storageManager.isStorageCapacityCalculated()) { + continue; + } + + // Periodically delete any unnecessary files from the temp directory + if (offset == 0 || offset % (limit * 10) == 0) { + this.cleanupTempDirectory(now); + } + + // Any arbitrary transactions we want to fetch data for? + try (final Repository repository = RepositoryManager.getRepository()) { + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true); + // LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit); + if (isStopping) { + return; + } + + if (signatures == null || signatures.isEmpty()) { + offset = 0; + continue; + } + offset += limit; + now = NTP.getTime(); + + // Loop through the signatures in this batch + for (int i=0; i findPathsWithNoAssociatedTransaction(Repository repository) { + List pathList = new ArrayList<>(); + + // Find all hosted paths + List allPaths = ArbitraryDataStorageManager.getInstance().findAllHostedPaths(); + + // Loop through each path and find those without matching signatures + for (Path path : allPaths) { + if (isStopping) { + break; + } + try { + String[] contents = path.toFile().list(); + if (contents == null || contents.length == 0) { + // Ignore empty directories + continue; + } + + String signature58 = path.getFileName().toString(); + byte[] signature = Base58.decode(signature58); + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (transactionData == null) { + // No transaction data, and no DataException, so we can assume that this data relates to an expired transaction + pathList.add(path); + } + + } catch (DataException e) { + continue; + } + } + + return pathList; + } + + private void checkForExpiredTransactions(Repository repository) { + List expiredPaths = this.findPathsWithNoAssociatedTransaction(repository); + for (Path expiredPath : expiredPaths) { + if (isStopping) { + return; + } + LOGGER.info("Found path with no associated transaction: {}", expiredPath.toString()); + this.safeDeleteDirectory(expiredPath.toFile(), "no matching transaction"); + } + } + + private void storageLimitReached(Repository repository) throws InterruptedException { + // We think that the storage limit has been reached + + // Now calculate the used/total storage again, as a safety precaution + Long now = NTP.getTime(); + ArbitraryDataStorageManager.getInstance().calculateDirectorySize(now); + if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailable(DELETION_THRESHOLD)) { + // We have space available, so don't delete anything + return; + } + + // Delete a batch of random chunks + // This reduces the chance of too many nodes deleting the same chunk + // when they reach their storage limit + Path dataPath = Paths.get(Settings.getInstance().getDataPath()); + for (int i=0; i + * Key is original request's message ID
+ * Value is Triple<transaction signature in base58, first requesting peer, first request's timestamp> + *

+ * If peer is null then either:
+ *

    + *
  • we are the original requesting peer
    • + *
  • we have already sent data payload to original requesting peer.
    • + *
+ * If signature is null then we have already received the file list and either:
+ *
    + *
  • we are the original requesting peer and have processed it
    • + *
  • we have forwarded the file list
    • + *
+ */ + public Map> arbitraryDataFileListRequests = Collections.synchronizedMap(new HashMap<>()); + + /** + * Map to keep track of in progress arbitrary data signature requests + * Key: string - the signature encoded in base58 + * Value: Triple + */ + private Map> arbitraryDataSignatureRequests = Collections.synchronizedMap(new HashMap<>()); + + + /** Maximum number of seconds that a file list relay request is able to exist on the network */ + private static long RELAY_REQUEST_MAX_DURATION = 5000L; + /** Maximum number of hops that a file list relay request is allowed to make */ + private static int RELAY_REQUEST_MAX_HOPS = 4; + + + private ArbitraryDataFileListManager() { + } + + public static ArbitraryDataFileListManager getInstance() { + if (instance == null) + instance = new ArbitraryDataFileListManager(); + + return instance; + } + + + public void cleanupRequestCache(Long now) { + if (now == null) { + return; + } + final long requestMinimumTimestamp = now - ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT; + arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < requestMinimumTimestamp); + } + + + // Track file list lookups by signature + + private boolean shouldMakeFileListRequestForSignature(String signature58) { + Triple request = arbitraryDataSignatureRequests.get(signature58); + + if (request == null) { + // Not attempted yet + return true; + } + + // Extract the components + Integer networkBroadcastCount = request.getA(); + // Integer directPeerRequestCount = request.getB(); + Long lastAttemptTimestamp = request.getC(); + + if (lastAttemptTimestamp == null) { + // Not attempted yet + return true; + } + + long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp; + + // Allow a second attempt after 15 seconds, and another after 30 seconds + if (timeSinceLastAttempt > 15 * 1000L) { + // We haven't tried for at least 15 seconds + + if (networkBroadcastCount < 3) { + // We've made less than 3 total attempts + return true; + } + } + + // Then allow another 5 attempts, each 5 minutes apart + if (timeSinceLastAttempt > 5 * 60 * 1000L) { + // We haven't tried for at least 5 minutes + + if (networkBroadcastCount < 5) { + // We've made less than 5 total attempts + return true; + } + } + + // From then on, only try once every 24 hours, to reduce network spam + if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) { + // We haven't tried for at least 24 hours + return true; + } + + return false; + } + + private boolean shouldMakeDirectFileRequestsForSignature(String signature58) { + if (!Settings.getInstance().isDirectDataRetrievalEnabled()) { + // Direct connections are disabled in the settings + return false; + } + + Triple request = arbitraryDataSignatureRequests.get(signature58); + + if (request == null) { + // Not attempted yet + return true; + } + + // Extract the components + //Integer networkBroadcastCount = request.getA(); + Integer directPeerRequestCount = request.getB(); + Long lastAttemptTimestamp = request.getC(); + + if (lastAttemptTimestamp == null) { + // Not attempted yet + return true; + } + + if (directPeerRequestCount == 0) { + // We haven't tried asking peers directly yet, so we should + return true; + } + + long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp; + if (timeSinceLastAttempt > 10 * 1000L) { + // We haven't tried for at least 10 seconds + if (directPeerRequestCount < 5) { + // We've made less than 5 total attempts + return true; + } + } + + if (timeSinceLastAttempt > 5 * 60 * 1000L) { + // We haven't tried for at least 5 minutes + if (directPeerRequestCount < 10) { + // We've made less than 10 total attempts + return true; + } + } + + if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) { + // We haven't tried for at least 24 hours + return true; + } + + return false; + } + + public boolean isSignatureRateLimited(byte[] signature) { + String signature58 = Base58.encode(signature); + return !this.shouldMakeFileListRequestForSignature(signature58) + && !this.shouldMakeDirectFileRequestsForSignature(signature58); + } + + public long lastRequestForSignature(byte[] signature) { + String signature58 = Base58.encode(signature); + Triple request = arbitraryDataSignatureRequests.get(signature58); + + if (request == null) { + // Not attempted yet + return 0; + } + + // Extract the components + Long lastAttemptTimestamp = request.getC(); + if (lastAttemptTimestamp != null) { + return lastAttemptTimestamp; + } + return 0; + } + + public void addToSignatureRequests(String signature58, boolean incrementNetworkRequests, boolean incrementPeerRequests) { + Triple request = arbitraryDataSignatureRequests.get(signature58); + Long now = NTP.getTime(); + + if (request == null) { + // No entry yet + Triple newRequest = new Triple<>(0, 0, now); + arbitraryDataSignatureRequests.put(signature58, newRequest); + } + else { + // There is an existing entry + if (incrementNetworkRequests) { + request.setA(request.getA() + 1); + } + if (incrementPeerRequests) { + request.setB(request.getB() + 1); + } + request.setC(now); + arbitraryDataSignatureRequests.put(signature58, request); + } + } + + public void removeFromSignatureRequests(String signature58) { + arbitraryDataSignatureRequests.remove(signature58); + } + + + // Lookup file lists by signature (and optionally hashes) + + public boolean fetchArbitraryDataFileList(ArbitraryTransactionData arbitraryTransactionData) { + byte[] digest = arbitraryTransactionData.getData(); + byte[] metadataHash = arbitraryTransactionData.getMetadataHash(); + byte[] signature = arbitraryTransactionData.getSignature(); + String signature58 = Base58.encode(signature); + + // Require an NTP sync + Long now = NTP.getTime(); + if (now == null) { + return false; + } + + // If we've already tried too many times in a short space of time, make sure to give up + if (!this.shouldMakeFileListRequestForSignature(signature58)) { + // Check if we should make direct connections to peers + if (this.shouldMakeDirectFileRequestsForSignature(signature58)) { + return ArbitraryDataFileManager.getInstance().fetchDataFilesFromPeersForSignature(signature); + } + + LOGGER.trace("Skipping file list request for signature {} due to rate limit", signature58); + return false; + } + this.addToSignatureRequests(signature58, true, false); + + List handshakedPeers = Network.getInstance().getHandshakedPeers(); + List missingHashes = null; + + // Find hashes that we are missing + try { + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + missingHashes = arbitraryDataFile.missingHashes(); + } catch (DataException e) { + // Leave missingHashes as null, so that all hashes are requested + } + int hashCount = missingHashes != null ? missingHashes.size() : 0; + + LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size())); + + // Build request + Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, missingHashes, now, 0); + + // Save our request into requests map + Triple requestEntry = new Triple<>(signature58, null, NTP.getTime()); + + // Assign random ID to this message + int id; + do { + id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1; + + // Put queue into map (keyed by message ID) so we can poll for a response + // If putIfAbsent() doesn't return null, then this ID is already taken + } while (arbitraryDataFileListRequests.put(id, requestEntry) != null); + getArbitraryDataFileListMessage.setId(id); + + // Broadcast request + Network.getInstance().broadcast(peer -> getArbitraryDataFileListMessage); + + // Poll to see if data has arrived + final long singleWait = 100; + long totalWait = 0; + while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) { + try { + Thread.sleep(singleWait); + } catch (InterruptedException e) { + break; + } + + requestEntry = arbitraryDataFileListRequests.get(id); + if (requestEntry == null) + return false; + + if (requestEntry.getA() == null) + break; + + totalWait += singleWait; + } + return true; + } + + public boolean fetchArbitraryDataFileList(Peer peer, byte[] signature) { + String signature58 = Base58.encode(signature); + + // Require an NTP sync + Long now = NTP.getTime(); + if (now == null) { + return false; + } + + int hashCount = 0; + LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to peer %s...", signature58, hashCount, peer)); + + // Build request + // Use a time in the past, so that the recipient peer doesn't try and relay it + // Also, set hashes to null since it's easier to request all hashes than it is to determine which ones we need + // This could be optimized in the future + long timestamp = now - 60000L; + List hashes = null; + Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, timestamp, 0); + + // Save our request into requests map + Triple requestEntry = new Triple<>(signature58, null, NTP.getTime()); + + // Assign random ID to this message + int id; + do { + id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1; + + // Put queue into map (keyed by message ID) so we can poll for a response + // If putIfAbsent() doesn't return null, then this ID is already taken + } while (arbitraryDataFileListRequests.put(id, requestEntry) != null); + getArbitraryDataFileListMessage.setId(id); + + // Send the request + peer.sendMessage(getArbitraryDataFileListMessage); + + // Poll to see if data has arrived + final long singleWait = 100; + long totalWait = 0; + while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) { + try { + Thread.sleep(singleWait); + } catch (InterruptedException e) { + break; + } + + requestEntry = arbitraryDataFileListRequests.get(id); + if (requestEntry == null) + return false; + + if (requestEntry.getA() == null) + break; + + totalWait += singleWait; + } + return true; + } + + public void deleteFileListRequestsForSignature(byte[] signature) { + String signature58 = Base58.encode(signature); + for (Iterator>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) { + Map.Entry> entry = it.next(); + if (entry == null || entry.getKey() == null || entry.getValue() != null) { + continue; + } + if (Objects.equals(entry.getValue().getA(), signature58)) { + // Update requests map to reflect that we've received all chunks + Triple newEntry = new Triple<>(null, null, entry.getValue().getC()); + arbitraryDataFileListRequests.put(entry.getKey(), newEntry); + } + } + } + + // Network handlers + + public void onNetworkArbitraryDataFileListMessage(Peer peer, Message message) { + // Don't process if QDN is disabled + if (!Settings.getInstance().isQdnEnabled()) { + return; + } + + ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; + LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size()); + + // Do we have a pending request for this data? + Triple request = arbitraryDataFileListRequests.get(message.getId()); + if (request == null || request.getA() == null) { + return; + } + boolean isRelayRequest = (request.getB() != null); + + // Does this message's signature match what we're expecting? + byte[] signature = arbitraryDataFileListMessage.getSignature(); + String signature58 = Base58.encode(signature); + if (!request.getA().equals(signature58)) { + return; + } + + List hashes = arbitraryDataFileListMessage.getHashes(); + if (hashes == null || hashes.isEmpty()) { + return; + } + + ArbitraryTransactionData arbitraryTransactionData = null; + ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance(); + + // Check transaction exists and hashes are correct + try (final Repository repository = RepositoryManager.getRepository()) { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (!(transactionData instanceof ArbitraryTransactionData)) + return; + + arbitraryTransactionData = (ArbitraryTransactionData) transactionData; + + // Load data file(s) + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature); + arbitraryDataFile.setMetadataHash(arbitraryTransactionData.getMetadataHash()); + +// // Check all hashes exist +// for (byte[] hash : hashes) { +// //LOGGER.debug("Received hash {}", Base58.encode(hash)); +// if (!arbitraryDataFile.containsChunk(hash)) { +// // Check the hash against the complete file +// if (!Arrays.equals(arbitraryDataFile.getHash(), hash)) { +// LOGGER.info("Received non-matching chunk hash {} for signature {}. This could happen if we haven't obtained the metadata file yet.", Base58.encode(hash), signature58); +// return; +// } +// } +// } + + if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) { + // Keep track of the hashes this peer reports to have access to + Long now = NTP.getTime(); + for (byte[] hash : hashes) { + String hash58 = Base58.encode(hash); + String sig58 = Base58.encode(signature); + ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.put(hash58, new Triple<>(peer, sig58, now)); + } + + // Go and fetch the actual data, since this isn't a relay request + arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, hashes); + } + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e); + } + + // Forwarding + if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) { + boolean isBlocked = (arbitraryTransactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(arbitraryTransactionData.getName())); + if (!isBlocked) { + Peer requestingPeer = request.getB(); + if (requestingPeer != null) { + // Add each hash to our local mapping so we know who to ask later + Long now = NTP.getTime(); + for (byte[] hash : hashes) { + String hash58 = Base58.encode(hash); + ArbitraryRelayInfo relayMap = new ArbitraryRelayInfo(hash58, signature58, peer, now); + ArbitraryDataFileManager.getInstance().addToRelayMap(relayMap); + } + + // Forward to requesting peer + LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer); + if (!requestingPeer.sendMessage(arbitraryDataFileListMessage)) { + requestingPeer.disconnect("failed to forward arbitrary data file list"); + } + } + } + } + } + + public void onNetworkGetArbitraryDataFileListMessage(Peer peer, Message message) { + // Don't respond if QDN is disabled + if (!Settings.getInstance().isQdnEnabled()) { + return; + } + + Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet(); + + GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; + byte[] signature = getArbitraryDataFileListMessage.getSignature(); + String signature58 = Base58.encode(signature); + List requestedHashes = getArbitraryDataFileListMessage.getHashes(); + Long now = NTP.getTime(); + Triple newEntry = new Triple<>(signature58, peer, now); + + // If we've seen this request recently, then ignore + if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) { + LOGGER.debug("Ignoring hash list request from peer {} for signature {}", peer, signature58); + return; + } + + LOGGER.debug("Received hash list request from peer {} for signature {}", peer, signature58); + + List hashes = new ArrayList<>(); + ArbitraryTransactionData transactionData = null; + boolean allChunksExist = false; + + try (final Repository repository = RepositoryManager.getRepository()) { + + // Firstly we need to lookup this file on chain to get a list of its hashes + transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature); + if (transactionData instanceof ArbitraryTransactionData) { + + // Check if we're even allowed to serve data for this transaction + if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) { + + byte[] hash = transactionData.getData(); + byte[] metadataHash = transactionData.getMetadataHash(); + + // Load file(s) and add any that exist to the list of hashes + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + + // If the peer didn't supply a hash list, we need to return all hashes for this transaction + if (requestedHashes == null || requestedHashes.isEmpty()) { + requestedHashes = new ArrayList<>(); + + // Add the metadata file + if (arbitraryDataFile.getMetadataHash() != null) { + requestedHashes.add(arbitraryDataFile.getMetadataHash()); + } + + // Add the chunk hashes + if (arbitraryDataFile.getChunkHashes().size() > 0) { + requestedHashes.addAll(arbitraryDataFile.getChunkHashes()); + } + // Add complete file if there are no hashes + else { + requestedHashes.add(arbitraryDataFile.getHash()); + } + } + + // Assume all chunks exists, unless one can't be found below + allChunksExist = true; + + for (byte[] requestedHash : requestedHashes) { + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature); + if (chunk.exists()) { + hashes.add(chunk.getHash()); + //LOGGER.trace("Added hash {}", chunk.getHash58()); + } else { + LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58()); + allChunksExist = false; + } + } + } + } + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e); + } + + // We should only respond if we have at least one hash + if (hashes.size() > 0) { + + // We have all the chunks, so update requests map to reflect that we've sent it + // There is no need to keep track of the request, as we can serve all the chunks + if (allChunksExist) { + newEntry = new Triple<>(null, null, now); + arbitraryDataFileListRequests.put(message.getId(), newEntry); + } + + ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + arbitraryDataFileListMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryDataFileListMessage)) { + LOGGER.debug("Couldn't send list of hashes"); + peer.disconnect("failed to send list of hashes"); + return; + } + LOGGER.debug("Sent list of hashes (count: {})", hashes.size()); + + if (allChunksExist) { + // Nothing left to do, so return to prevent any unnecessary forwarding from occurring + LOGGER.debug("No need for any forwarding because file list request is fully served"); + return; + } + + } + + // We may need to forward this request on + boolean isBlocked = (transactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName())); + if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { + // In relay mode - so ask our other peers if they have it + + long requestTime = getArbitraryDataFileListMessage.getRequestTime(); + int requestHops = getArbitraryDataFileListMessage.getRequestHops(); + getArbitraryDataFileListMessage.setRequestHops(++requestHops); + long totalRequestTime = now - requestTime; + + if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { + // Relay request hasn't timed out yet, so can potentially be rebroadcast + if (requestHops < RELAY_REQUEST_MAX_HOPS) { + // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast + + LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); + Network.getInstance().broadcast( + broadcastPeer -> broadcastPeer == peer || + Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) + ? null : getArbitraryDataFileListMessage); + + } + else { + // This relay request has reached the maximum number of allowed hops + } + } + else { + // This relay request has timed out + } + } + } + +} diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java new file mode 100644 index 000000000..9b437e2b7 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -0,0 +1,500 @@ +package org.qortal.controller.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryRelayInfo; +import org.qortal.data.network.ArbitraryPeerData; +import org.qortal.data.network.PeerData; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.*; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.utils.ArbitraryTransactionUtils; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; +import org.qortal.utils.Triple; + +import java.security.SecureRandom; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +public class ArbitraryDataFileManager extends Thread { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileManager.class); + + private static ArbitraryDataFileManager instance; + private volatile boolean isStopping = false; + + + /** + * Map to keep track of our in progress (outgoing) arbitrary data file requests + */ + public Map arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>()); + + /** + * Map to keep track of hashes that we might need to relay + */ + public List arbitraryRelayMap = Collections.synchronizedList(new ArrayList<>()); + + /** + * Map to keep track of any arbitrary data file hash responses + * Key: string - the hash encoded in base58 + * Value: Triple + */ + public Map> arbitraryDataFileHashResponses = Collections.synchronizedMap(new HashMap<>()); + + + private ArbitraryDataFileManager() { + } + + public static ArbitraryDataFileManager getInstance() { + if (instance == null) + instance = new ArbitraryDataFileManager(); + + return instance; + } + + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data File Manager"); + + try { + // Use a fixed thread pool to execute the arbitrary data file requests + int threadCount = 10; + ExecutorService arbitraryDataFileRequestExecutor = Executors.newFixedThreadPool(threadCount); + for (int i = 0; i < threadCount; i++) { + arbitraryDataFileRequestExecutor.execute(new ArbitraryDataFileRequestThread()); + } + + while (!isStopping) { + // Nothing to do yet + Thread.sleep(1000); + } + } catch (InterruptedException e) { + // Fall-through to exit thread... + } + } + + public void shutdown() { + isStopping = true; + this.interrupt(); + } + + + public void cleanupRequestCache(Long now) { + if (now == null) { + return; + } + final long requestMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_REQUEST_TIMEOUT; + arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < requestMinimumTimestamp); + + final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT; + arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp); + arbitraryDataFileHashResponses.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < relayMinimumTimestamp); + } + + + + // Fetch data files by hash + + public boolean fetchArbitraryDataFiles(Repository repository, + Peer peer, + byte[] signature, + ArbitraryTransactionData arbitraryTransactionData, + List hashes) throws DataException { + + // Load data file(s) + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature); + byte[] metadataHash = arbitraryTransactionData.getMetadataHash(); + arbitraryDataFile.setMetadataHash(metadataHash); + boolean receivedAtLeastOneFile = false; + + // Now fetch actual data from this peer + for (byte[] hash : hashes) { + if (isStopping) { + return false; + } + String hash58 = Base58.encode(hash); + if (!arbitraryDataFile.chunkExists(hash)) { + // Only request the file if we aren't already requesting it from someone else + if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) { + LOGGER.debug("Requesting data file {} from peer {}", hash58, peer); + Long startTime = NTP.getTime(); + ArbitraryDataFileMessage receivedArbitraryDataFileMessage = fetchArbitraryDataFile(peer, null, signature, hash, null); + Long endTime = NTP.getTime(); + if (receivedArbitraryDataFileMessage != null) { + LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFileMessage.getArbitraryDataFile().getHash58(), peer, (endTime-startTime)); + receivedAtLeastOneFile = true; + + // Remove this hash from arbitraryDataFileHashResponses now that we have received it + arbitraryDataFileHashResponses.remove(hash58); + } + else { + LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime)); + + // Remove this hash from arbitraryDataFileHashResponses now that we have failed to receive it + arbitraryDataFileHashResponses.remove(hash58); + + // Stop asking for files from this peer + break; + } + } + else { + LOGGER.trace("Already requesting data file {} for signature {} from peer {}", arbitraryDataFile, Base58.encode(signature), peer); + } + } + else { + // Remove this hash from arbitraryDataFileHashResponses because we have a local copy + arbitraryDataFileHashResponses.remove(hash58); + } + } + + if (receivedAtLeastOneFile) { + // Update our lookup table to indicate that this peer holds data for this signature + String peerAddress = peer.getPeerData().getAddress().toString(); + ArbitraryPeerData arbitraryPeerData = new ArbitraryPeerData(signature, peer); + repository.discardChanges(); + if (arbitraryPeerData.isPeerAddressValid()) { + LOGGER.debug("Adding arbitrary peer: {} for signature {}", peerAddress, Base58.encode(signature)); + repository.getArbitraryRepository().save(arbitraryPeerData); + repository.saveChanges(); + } + + // Invalidate the hosted transactions cache as we are now hosting something new + ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache(); + + // Check if we have all the files we need for this transaction + if (arbitraryDataFile.allFilesExist()) { + + // We have all the chunks for this transaction, so we should invalidate the transaction's name's + // data cache so that it is rebuilt the next time we serve it + ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData); + + // We may also need to broadcast to the network that we are now hosting files for this transaction, + // but only if these files are in accordance with our storage policy + if (ArbitraryDataStorageManager.getInstance().canStoreData(arbitraryTransactionData)) { + // Use a null peer address to indicate our own + Message newArbitrarySignatureMessage = new ArbitrarySignaturesMessage(null, 0, Arrays.asList(signature)); + Network.getInstance().broadcast(broadcastPeer -> newArbitrarySignatureMessage); + } + } + + } + + return receivedAtLeastOneFile; + } + + private ArbitraryDataFileMessage fetchArbitraryDataFile(Peer peer, Peer requestingPeer, byte[] signature, byte[] hash, Message originalMessage) throws DataException { + ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature); + boolean fileAlreadyExists = existingFile.exists(); + String hash58 = Base58.encode(hash); + Message message = null; + + // Fetch the file if it doesn't exist locally + if (!fileAlreadyExists) { + LOGGER.debug(String.format("Fetching data file %.8s from peer %s", hash58, peer)); + arbitraryDataFileRequests.put(hash58, NTP.getTime()); + Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash); + + try { + message = peer.getResponseWithTimeout(getArbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT); + } catch (InterruptedException e) { + // Will return below due to null message + } + arbitraryDataFileRequests.remove(hash58); + LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58)); + + // We may need to remove the file list request, if we have all the files for this transaction + this.handleFileListRequests(signature); + + if (message == null) { + LOGGER.debug("Received null message from peer {}", peer); + return null; + } + if (message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) { + LOGGER.debug("Received message with invalid type: {} from peer {}", message.getType(), peer); + return null; + } + } + else { + LOGGER.debug(String.format("File hash %s already exists, so skipping the request", hash58)); + } + ArbitraryDataFileMessage arbitraryDataFileMessage = (ArbitraryDataFileMessage) message; + + // We might want to forward the request to the peer that originally requested it + this.handleArbitraryDataFileForwarding(requestingPeer, message, originalMessage); + + boolean isRelayRequest = (requestingPeer != null); + if (isRelayRequest) { + if (!fileAlreadyExists) { + // File didn't exist locally before the request, and it's a forwarding request, so delete it + LOGGER.debug("Deleting file {} because it was needed for forwarding only", Base58.encode(hash)); + ArbitraryDataFile dataFile = arbitraryDataFileMessage.getArbitraryDataFile(); + + // Keep trying to delete the data until it is deleted, or we reach 10 attempts + dataFile.delete(10); + } + } + + return arbitraryDataFileMessage; + } + + private void handleFileListRequests(byte[] signature) { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Fetch the transaction data + ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); + if (arbitraryTransactionData == null) { + return; + } + + boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData); + + if (allChunksExist) { + // Update requests map to reflect that we've received all chunks + ArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature); + } + + } catch (DataException e) { + LOGGER.debug("Unable to handle file list requests: {}", e.getMessage()); + } + } + + public void handleArbitraryDataFileForwarding(Peer requestingPeer, Message message, Message originalMessage) { + // Return if there is no originally requesting peer to forward to + if (requestingPeer == null) { + return; + } + + // Return if we're not in relay mode or if this request doesn't need forwarding + if (!Settings.getInstance().isRelayModeEnabled()) { + return; + } + + LOGGER.debug("Received arbitrary data file - forwarding is needed"); + + // The ID needs to match that of the original request + message.setId(originalMessage.getId()); + + if (!requestingPeer.sendMessage(message)) { + LOGGER.debug("Failed to forward arbitrary data file to peer {}", requestingPeer); + requestingPeer.disconnect("failed to forward arbitrary data file"); + } + else { + LOGGER.debug("Forwarded arbitrary data file to peer {}", requestingPeer); + } + } + + + // Fetch data directly from peers + + public boolean fetchDataFilesFromPeersForSignature(byte[] signature) { + String signature58 = Base58.encode(signature); + ArbitraryDataFileListManager.getInstance().addToSignatureRequests(signature58, false, true); + + // Firstly fetch peers that claim to be hosting files for this signature + try (final Repository repository = RepositoryManager.getRepository()) { + + List peers = repository.getArbitraryRepository().getArbitraryPeerDataForSignature(signature); + if (peers == null || peers.isEmpty()) { + LOGGER.debug("No peers found for signature {}", signature58); + return false; + } + + LOGGER.debug("Attempting a direct peer connection for signature {}...", signature58); + + // Peers found, so pick a random one and request data from it + int index = new SecureRandom().nextInt(peers.size()); + ArbitraryPeerData arbitraryPeerData = peers.get(index); + String peerAddressString = arbitraryPeerData.getPeerAddress(); + boolean success = Network.getInstance().requestDataFromPeer(peerAddressString, signature); + + // Parse the peer address to find the host and port + String host = null; + int port = -1; + String[] parts = peerAddressString.split(":"); + if (parts.length > 1) { + host = parts[0]; + port = Integer.parseInt(parts[1]); + } + + // If unsuccessful, and using a non-standard port, try a second connection with the default listen port, + // since almost all nodes use that. This is a workaround to account for any ephemeral ports that may + // have made it into the dataset. + if (!success) { + if (host != null && port > 0) { + int defaultPort = Settings.getInstance().getDefaultListenPort(); + if (port != defaultPort) { + String newPeerAddressString = String.format("%s:%d", host, defaultPort); + success = Network.getInstance().requestDataFromPeer(newPeerAddressString, signature); + } + } + } + + // If _still_ unsuccessful, try matching the peer's IP address with some known peers, and then connect + // to each of those in turn until one succeeds. + if (!success) { + if (host != null) { + final String finalHost = host; + List knownPeers = Network.getInstance().getAllKnownPeers().stream() + .filter(knownPeerData -> knownPeerData.getAddress().getHost().equals(finalHost)) + .collect(Collectors.toList()); + // Loop through each match and attempt a connection + for (PeerData matchingPeer : knownPeers) { + String matchingPeerAddress = matchingPeer.getAddress().toString(); + success = Network.getInstance().requestDataFromPeer(matchingPeerAddress, signature); + if (success) { + // Successfully connected, so stop making connections + break; + } + } + } + } + + // Keep track of the success or failure + arbitraryPeerData.markAsAttempted(); + if (success) { + arbitraryPeerData.markAsRetrieved(); + arbitraryPeerData.incrementSuccesses(); + } + else { + arbitraryPeerData.incrementFailures(); + } + repository.discardChanges(); + repository.getArbitraryRepository().save(arbitraryPeerData); + repository.saveChanges(); + + return success; + + } catch (DataException e) { + LOGGER.debug("Unable to fetch peer list from repository"); + } + + return false; + } + + + // Relays + + private List getRelayInfoListForHash(String hash58) { + synchronized (arbitraryRelayMap) { + return arbitraryRelayMap.stream() + .filter(relayInfo -> Objects.equals(relayInfo.getHash58(), hash58)) + .collect(Collectors.toList()); + } + } + + private ArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) { + LOGGER.trace("Fetching random relay info for hash: {}", hash58); + List relayInfoList = this.getRelayInfoListForHash(hash58); + if (relayInfoList != null && !relayInfoList.isEmpty()) { + + // Pick random item + int index = new SecureRandom().nextInt(relayInfoList.size()); + LOGGER.trace("Returning random relay info for hash: {} (index {})", hash58, index); + return relayInfoList.get(index); + } + LOGGER.trace("No relay info exists for hash: {}", hash58); + return null; + } + + public void addToRelayMap(ArbitraryRelayInfo newEntry) { + if (newEntry == null || !newEntry.isValid()) { + return; + } + + // Remove existing entry for this peer if it exists, to renew the timestamp + this.removeFromRelayMap(newEntry); + + // Re-add + arbitraryRelayMap.add(newEntry); + LOGGER.debug("Added entry to relay map: {}", newEntry); + } + + private void removeFromRelayMap(ArbitraryRelayInfo entry) { + arbitraryRelayMap.removeIf(relayInfo -> relayInfo.equals(entry)); + } + + + // Network handlers + + public void onNetworkGetArbitraryDataFileMessage(Peer peer, Message message) { + // Don't respond if QDN is disabled + if (!Settings.getInstance().isQdnEnabled()) { + return; + } + + GetArbitraryDataFileMessage getArbitraryDataFileMessage = (GetArbitraryDataFileMessage) message; + byte[] hash = getArbitraryDataFileMessage.getHash(); + String hash58 = Base58.encode(hash); + byte[] signature = getArbitraryDataFileMessage.getSignature(); + Controller.getInstance().stats.getArbitraryDataFileMessageStats.requests.incrementAndGet(); + + LOGGER.debug("Received GetArbitraryDataFileMessage from peer {} for hash {}", peer, Base58.encode(hash)); + + try { + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); + ArbitraryRelayInfo relayInfo = this.getRandomRelayInfoEntryForHash(hash58); + + if (arbitraryDataFile.exists()) { + LOGGER.trace("Hash {} exists", hash58); + + // We can serve the file directly as we already have it + ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, arbitraryDataFile); + arbitraryDataFileMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryDataFileMessage)) { + LOGGER.debug("Couldn't sent file"); + peer.disconnect("failed to send file"); + } + LOGGER.debug("Sent file {}", arbitraryDataFile); + } + else if (relayInfo != null) { + LOGGER.debug("We have relay info for hash {}", Base58.encode(hash)); + // We need to ask this peer for the file + Peer peerToAsk = relayInfo.getPeer(); + if (peerToAsk != null) { + + // Forward the message to this peer + LOGGER.debug("Asking peer {} for hash {}", peerToAsk, hash58); + this.fetchArbitraryDataFile(peerToAsk, peer, signature, hash, message); + } + else { + LOGGER.debug("Peer {} not found in relay info", peer); + } + } + else { + LOGGER.debug("Hash {} doesn't exist and we don't have relay info", hash58); + + // We don't have this file + Controller.getInstance().stats.getArbitraryDataFileMessageStats.unknownFiles.getAndIncrement(); + + // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout + LOGGER.debug(String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile)); + + // We'll send empty block summaries message as it's very short + // TODO: use a different message type here + Message fileUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + fileUnknownMessage.setId(message.getId()); + if (!peer.sendMessage(fileUnknownMessage)) { + LOGGER.debug("Couldn't sent file-unknown response"); + peer.disconnect("failed to send file-unknown response"); + } + else { + LOGGER.debug("Sent file-unknown response for file {}", arbitraryDataFile); + } + } + } + catch (DataException e) { + LOGGER.debug("Unable to handle request for arbitrary data file: {}", hash58); + } + } + +} diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java new file mode 100644 index 000000000..e46fd2fbe --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -0,0 +1,123 @@ +package org.qortal.controller.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.network.Peer; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.ArbitraryTransactionUtils; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; +import org.qortal.utils.Triple; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; + +public class ArbitraryDataFileRequestThread implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileRequestThread.class); + + public ArbitraryDataFileRequestThread() { + + } + + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data File Request Thread"); + + try { + while (!Controller.isStopping()) { + Long now = NTP.getTime(); + this.processFileHashes(now); + } + } catch (InterruptedException e) { + // Fall-through to exit thread... + } + } + + private void processFileHashes(Long now) throws InterruptedException { + if (Controller.isStopping()) { + return; + } + + ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance(); + String signature58 = null; + String hash58 = null; + Peer peer = null; + boolean shouldProcess = false; + + synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) { + Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.entrySet().iterator(); + while (iterator.hasNext()) { + if (Controller.isStopping()) { + return; + } + + Map.Entry entry = (Map.Entry) iterator.next(); + if (entry == null || entry.getKey() == null || entry.getValue() == null) { + iterator.remove(); + continue; + } + + hash58 = (String) entry.getKey(); + Triple value = (Triple) entry.getValue(); + if (value == null) { + iterator.remove(); + continue; + } + + peer = value.getA(); + signature58 = value.getB(); + Long timestamp = value.getC(); + + if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { + // Ignore - to be deleted + iterator.remove(); + continue; + } + + // Skip if already requesting, but don't remove, as we might want to retry later + if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) { + // Already requesting - leave this attempt for later + continue; + } + + // We want to process this file + shouldProcess = true; + iterator.remove(); + break; + } + } + + if (!shouldProcess) { + // Nothing to do + Thread.sleep(1000L); + return; + } + + byte[] hash = Base58.decode(hash58); + byte[] signature = Base58.decode(signature58); + + // Fetch the transaction data + try (final Repository repository = RepositoryManager.getRepository()) { + ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); + if (arbitraryTransactionData == null) { + return; + } + + if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) { + return; + } + + LOGGER.debug("Fetching file {} from peer {} via request thread...", hash58, peer); + arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash)); + + } catch (DataException e) { + LOGGER.debug("Unable to process file hashes: {}", e.getMessage()); + } + } +} diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java new file mode 100644 index 000000000..acde16eb1 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -0,0 +1,461 @@ +package org.qortal.controller.arbitrary; + +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataResource; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.Controller; +import org.qortal.data.network.ArbitraryPeerData; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.list.ResourceListManager; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.*; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.transaction.ArbitraryTransaction; +import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.utils.ArbitraryTransactionUtils; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +public class ArbitraryDataManager extends Thread { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataManager.class); + private static final List ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY); + + /** Difficulty (leading zero bits) used in arbitrary data transactions + * Set here so that it can be more easily reduced when running unit tests */ + private int powDifficulty = 14; // Must not be final, as unit tests need to reduce this value + + /** Request timeout when transferring arbitrary data */ + public static final long ARBITRARY_REQUEST_TIMEOUT = 12 * 1000L; // ms + + /** Maximum time to hold information about an in-progress relay */ + public static final long ARBITRARY_RELAY_TIMEOUT = 60 * 1000L; // ms + + /** Maximum number of hops that an arbitrary signatures request is allowed to make */ + private static int ARBITRARY_SIGNATURES_REQUEST_MAX_HOPS = 3; + + private static ArbitraryDataManager instance; + private final Object peerDataLock = new Object(); + + private volatile boolean isStopping = false; + + /** + * Map to keep track of cached arbitrary transaction resources. + * When an item is present in this list with a timestamp in the future, we won't invalidate + * its cache when serving that data. This reduces the amount of database lookups that are needed. + */ + private Map arbitraryDataCachedResources = Collections.synchronizedMap(new HashMap<>()); + + /** + * The amount of time to cache a data resource before it is invalidated + */ + private static long ARBITRARY_DATA_CACHE_TIMEOUT = 60 * 60 * 1000L; // 60 minutes + + + + private ArbitraryDataManager() { + } + + public static ArbitraryDataManager getInstance() { + if (instance == null) + instance = new ArbitraryDataManager(); + + return instance; + } + + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data Manager"); + + try { + // Wait for node to finish starting up and making connections + Thread.sleep(2 * 60 * 1000L); + + while (!isStopping) { + Thread.sleep(2000); + + // Don't run if QDN is disabled + if (!Settings.getInstance().isQdnEnabled()) { + Thread.sleep(60 * 60 * 1000L); + continue; + } + + List peers = Network.getInstance().getHandshakedPeers(); + + // Disregard peers that have "misbehaved" recently + peers.removeIf(Controller.hasMisbehaved); + + // Don't fetch data if we don't have enough up-to-date peers + if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) { + continue; + } + + // Fetch data according to storage policy + switch (Settings.getInstance().getStoragePolicy()) { + case FOLLOWED: + case FOLLOWED_OR_VIEWED: + this.processNames(); + break; + + case ALL: + this.processAll(); + + case NONE: + case VIEWED: + default: + // Nothing to fetch in advance + Thread.sleep(60000); + break; + } + } + } catch (InterruptedException e) { + // Fall-through to exit thread... + } + } + + public void shutdown() { + isStopping = true; + this.interrupt(); + } + + private void processNames() { + // Fetch latest list of followed names + List followedNames = ResourceListManager.getInstance().getStringsInList("followedNames"); + if (followedNames == null || followedNames.isEmpty()) { + return; + } + + // Loop through the names in the list and fetch transactions for each + for (String name : followedNames) { + this.fetchAndProcessTransactions(name); + } + } + + private void processAll() { + this.fetchAndProcessTransactions(null); + } + + private void fetchAndProcessTransactions(String name) { + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + + // Paginate queries when fetching arbitrary transactions + final int limit = 100; + int offset = 0; + + while (!isStopping) { + + // Any arbitrary transactions we want to fetch data for? + try (final Repository repository = RepositoryManager.getRepository()) { + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, name, null, ConfirmationStatus.BOTH, limit, offset, true); + // LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit); + if (signatures == null || signatures.isEmpty()) { + offset = 0; + break; + } + offset += limit; + + // Loop through signatures and remove ones we don't need to process + Iterator iterator = signatures.iterator(); + while (iterator.hasNext()) { + byte[] signature = (byte[]) iterator.next(); + + ArbitraryTransaction arbitraryTransaction = fetchTransaction(repository, signature); + if (arbitraryTransaction == null) { + // Best not to process this one + iterator.remove(); + continue; + } + ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData(); + + // Skip transactions that we don't need to proactively store data for + if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) { + iterator.remove(); + continue; + } + + // Remove transactions that we already have local data for + if (hasLocalData(arbitraryTransaction)) { + iterator.remove(); + continue; + } + } + + if (signatures.isEmpty()) { + continue; + } + + // Pick one at random + final int index = new Random().nextInt(signatures.size()); + byte[] signature = signatures.get(index); + + if (signature == null) { + continue; + } + + // Check to see if we have had a more recent PUT + ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); + boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData); + if (hasMoreRecentPutTransaction) { + // There is a more recent PUT transaction than the one we are currently processing. + // When a PUT is issued, it replaces any layers that would have been there before. + // Therefore any data relating to this older transaction is no longer needed and we + // shouldn't fetch it from the network. + continue; + } + + // Ask our connected peers if they have files for this signature + // This process automatically then fetches the files themselves if a peer is found + fetchData(arbitraryTransactionData); + + } catch (DataException e) { + LOGGER.error("Repository issue when fetching arbitrary transaction data", e); + } + } + } + + private ArbitraryTransaction fetchTransaction(final Repository repository, byte[] signature) { + try { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (!(transactionData instanceof ArbitraryTransactionData)) + return null; + + return new ArbitraryTransaction(repository, transactionData); + + } catch (DataException e) { + return null; + } + } + + private boolean hasLocalData(ArbitraryTransaction arbitraryTransaction) { + try { + return arbitraryTransaction.isDataLocal(); + + } catch (DataException e) { + LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e); + return true; + } + } + + + // Entrypoint to request new data from peers + public boolean fetchData(ArbitraryTransactionData arbitraryTransactionData) { + return ArbitraryDataFileListManager.getInstance().fetchArbitraryDataFileList(arbitraryTransactionData); + } + + + // Useful methods used by other parts of the app + + public boolean isSignatureRateLimited(byte[] signature) { + return ArbitraryDataFileListManager.getInstance().isSignatureRateLimited(signature); + } + + public long lastRequestForSignature(byte[] signature) { + return ArbitraryDataFileListManager.getInstance().lastRequestForSignature(signature); + } + + + // Arbitrary data resource cache + + public void cleanupRequestCache(Long now) { + if (now == null) { + return; + } + + // Cleanup file list request caches + ArbitraryDataFileListManager.getInstance().cleanupRequestCache(now); + + // Cleanup file request caches + ArbitraryDataFileManager.getInstance().cleanupRequestCache(now); + } + + public boolean isResourceCached(ArbitraryDataResource resource) { + if (resource == null) { + return false; + } + String key = resource.getUniqueKey(); + + // We don't have an entry for this resource ID, it is not cached + if (this.arbitraryDataCachedResources == null) { + return false; + } + if (!this.arbitraryDataCachedResources.containsKey(key)) { + return false; + } + Long timestamp = this.arbitraryDataCachedResources.get(key); + if (timestamp == null) { + return false; + } + + // If the timestamp has reached the timeout, we should remove it from the cache + long now = NTP.getTime(); + if (now > timestamp) { + this.arbitraryDataCachedResources.remove(key); + return false; + } + + // Current time hasn't reached the timeout, so treat it as cached + return true; + } + + public void addResourceToCache(ArbitraryDataResource resource) { + if (resource == null) { + return; + } + String key = resource.getUniqueKey(); + + // Just in case + if (this.arbitraryDataCachedResources == null) { + this.arbitraryDataCachedResources = new HashMap<>(); + } + + Long now = NTP.getTime(); + if (now == null) { + return; + } + + // Set the timestamp to now + the timeout + Long timestamp = NTP.getTime() + ARBITRARY_DATA_CACHE_TIMEOUT; + this.arbitraryDataCachedResources.put(key, timestamp); + } + + public void invalidateCache(ArbitraryTransactionData arbitraryTransactionData) { + String signature58 = Base58.encode(arbitraryTransactionData.getSignature()); + + if (arbitraryTransactionData.getName() != null) { + String resourceId = arbitraryTransactionData.getName().toLowerCase(); + Service service = arbitraryTransactionData.getService(); + String identifier = arbitraryTransactionData.getIdentifier(); + + ArbitraryDataResource resource = + new ArbitraryDataResource(resourceId, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); + String key = resource.getUniqueKey(); + LOGGER.trace("Clearing cache for {}...", resource); + + if (this.arbitraryDataCachedResources.containsKey(key)) { + this.arbitraryDataCachedResources.remove(key); + } + + // Also remove from the failed builds queue in case it previously failed due to missing chunks + ArbitraryDataBuildManager buildManager = ArbitraryDataBuildManager.getInstance(); + if (buildManager.arbitraryDataFailedBuilds.containsKey(key)) { + buildManager.arbitraryDataFailedBuilds.remove(key); + } + + // Remove from the signature requests list now that we have all files for this signature + ArbitraryDataFileListManager.getInstance().removeFromSignatureRequests(signature58); + + // Delete cached files themselves + try { + resource.deleteCache(); + } catch (IOException e) { + LOGGER.info("Unable to delete cache for resource {}: {}", resource, e.getMessage()); + } + } + } + + + // Broadcast list of hosted signatures + + public void broadcastHostedSignatureList() { + try (final Repository repository = RepositoryManager.getRepository()) { + List hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, null, null); + List hostedSignatures = hostedTransactions.stream().map(ArbitraryTransactionData::getSignature).collect(Collectors.toList()); + if (!hostedSignatures.isEmpty()) { + // Broadcast the list, using null to represent our peer address + LOGGER.info("Broadcasting list of hosted signatures..."); + Message arbitrarySignatureMessage = new ArbitrarySignaturesMessage(null, 0, hostedSignatures); + Network.getInstance().broadcast(broadcastPeer -> arbitrarySignatureMessage); + } + } catch (DataException e) { + LOGGER.error("Repository issue when fetching arbitrary transaction data for broadcast", e); + } + } + + + // Handle incoming arbitrary signatures messages + + public void onNetworkArbitrarySignaturesMessage(Peer peer, Message message) { + // Don't process if QDN is disabled + if (!Settings.getInstance().isQdnEnabled()) { + return; + } + + LOGGER.debug("Received arbitrary signature list from peer {}", peer); + + ArbitrarySignaturesMessage arbitrarySignaturesMessage = (ArbitrarySignaturesMessage) message; + List signatures = arbitrarySignaturesMessage.getSignatures(); + + String peerAddress = peer.getPeerData().getAddress().toString(); + if (arbitrarySignaturesMessage.getPeerAddress() != null && !arbitrarySignaturesMessage.getPeerAddress().isEmpty()) { + // This message is about a different peer than the one that sent it + peerAddress = arbitrarySignaturesMessage.getPeerAddress(); + } + + boolean containsNewEntry = false; + + // Synchronize peer data lookups to make this process thread safe. Otherwise we could broadcast + // the same data multiple times, due to more than one thread processing the same message from different peers + synchronized (this.peerDataLock) { + try (final Repository repository = RepositoryManager.getRepository()) { + for (byte[] signature : signatures) { + + // Check if a record already exists for this hash/host combination + // The port is not checked here - only the host/ip - in order to avoid duplicates + // from filling up the db due to dynamic/ephemeral ports + ArbitraryPeerData existingEntry = repository.getArbitraryRepository() + .getArbitraryPeerDataForSignatureAndHost(signature, peer.getPeerData().getAddress().getHost()); + + if (existingEntry == null) { + // We haven't got a record of this mapping yet, so add it + ArbitraryPeerData arbitraryPeerData = new ArbitraryPeerData(signature, peerAddress); + repository.discardChanges(); + if (arbitraryPeerData.isPeerAddressValid()) { + LOGGER.debug("Adding arbitrary peer: {} for signature {}", peerAddress, Base58.encode(signature)); + repository.getArbitraryRepository().save(arbitraryPeerData); + repository.saveChanges(); + + // Remember that this data is new, so that it can be rebroadcast later + containsNewEntry = true; + } + } + } + + // If at least one signature in this batch was new to us, we should rebroadcast the message to the + // network in case some peers haven't received it yet + if (containsNewEntry) { + int requestHops = arbitrarySignaturesMessage.getRequestHops(); + arbitrarySignaturesMessage.setRequestHops(++requestHops); + if (requestHops < ARBITRARY_SIGNATURES_REQUEST_MAX_HOPS) { + LOGGER.debug("Rebroadcasting arbitrary signature list for peer {}. requestHops: {}", peerAddress, requestHops); + Network.getInstance().broadcast(broadcastPeer -> broadcastPeer == peer ? null : arbitrarySignaturesMessage); + } + } else { + // Don't rebroadcast as otherwise we could get into a loop + } + + // If anything needed saving, it would already have called saveChanges() above + repository.discardChanges(); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while processing arbitrary transaction signature list from peer %s", peer), e); + } + } + } + + + public int getPowDifficulty() { + return this.powDifficulty; + } + +} diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java new file mode 100644 index 000000000..2844cef89 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java @@ -0,0 +1,86 @@ +package org.qortal.controller.arbitrary; + +import org.qortal.arbitrary.ArbitraryDataResource; +import org.qortal.utils.NTP; + +import java.util.*; + +public class ArbitraryDataRenderManager extends Thread { + + private static ArbitraryDataRenderManager instance; + private volatile boolean isStopping = false; + + /** + * Map to keep track of authorized resources for rendering. + * Keyed by resource ID, with the authorization time as the value. + */ + private Map authorizedResources = Collections.synchronizedMap(new HashMap<>()); + + private static long AUTHORIZATION_TIMEOUT = 60 * 60 * 1000L; // 1 hour + + + public ArbitraryDataRenderManager() { + + } + + public static ArbitraryDataRenderManager getInstance() { + if (instance == null) + instance = new ArbitraryDataRenderManager(); + + return instance; + } + + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data Render Manager"); + + try { + while (!isStopping) { + Thread.sleep(60000); + + Long now = NTP.getTime(); + this.cleanup(now); + } + } catch (InterruptedException e) { + // Fall-through to exit thread... + } + } + + public void shutdown() { + isStopping = true; + this.interrupt(); + } + + public void cleanup(Long now) { + if (now == null) { + return; + } + final long minimumTimestamp = now - AUTHORIZATION_TIMEOUT; + this.authorizedResources.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < minimumTimestamp); + } + + public boolean isAuthorized(ArbitraryDataResource resource) { + ArbitraryDataResource broadResource = new ArbitraryDataResource(resource.getResourceId(), null, null, null); + + for (String authorizedResourceKey : this.authorizedResources.keySet()) { + if (authorizedResourceKey != null && resource != null) { + // Check for exact match + if (Objects.equals(authorizedResourceKey, resource.getUniqueKey())) { + return true; + } + // Check for a broad authorization (which applies to all services and identifiers under an authorized name) + if (Objects.equals(authorizedResourceKey, broadResource.getUniqueKey())) { + return true; + } + } + } + return false; + } + + public void addToAuthorizedResources(ArbitraryDataResource resource) { + if (!this.isAuthorized(resource)) { + this.authorizedResources.put(resource.getUniqueKey(), NTP.getTime()); + } + } + +} diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java new file mode 100644 index 000000000..4b975b406 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -0,0 +1,557 @@ +package org.qortal.controller.arbitrary; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.list.ResourceListManager; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.settings.Settings; +import org.qortal.transaction.Transaction; +import org.qortal.utils.ArbitraryTransactionUtils; +import org.qortal.utils.Base58; +import org.qortal.utils.FilesystemUtils; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class ArbitraryDataStorageManager extends Thread { + + public enum StoragePolicy { + FOLLOWED_OR_VIEWED, + FOLLOWED, + VIEWED, + ALL, + NONE + } + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataStorageManager.class); + + private static ArbitraryDataStorageManager instance; + private volatile boolean isStopping = false; + + private Long storageCapacity = null; + private long totalDirectorySize = 0L; + private long lastDirectorySizeCheck = 0; + + private List hostedTransactions; + + private String searchQuery; + private List searchResultsTransactions; + + private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes + + /** Treat storage as full at 90% usage, to reduce risk of going over the limit. + * This is necessary because we don't calculate total storage values before every write. + * It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit. + * This must be lower than DELETION_THRESHOLD. */ + private static final double STORAGE_FULL_THRESHOLD = 0.90f; // 90% + + /** Start deleting files once we reach 98% usage. + * This must be higher than STORAGE_FULL_THRESHOLD in order to avoid a fetch/delete loop. */ + public static final double DELETION_THRESHOLD = 0.98f; // 98% + + public ArbitraryDataStorageManager() { + } + + public static ArbitraryDataStorageManager getInstance() { + if (instance == null) + instance = new ArbitraryDataStorageManager(); + + return instance; + } + + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data Storage Manager"); + try { + while (!isStopping) { + Thread.sleep(1000); + + // Don't run if QDN is disabled + if (!Settings.getInstance().isQdnEnabled()) { + Thread.sleep(60 * 60 * 1000L); + continue; + } + + Long now = NTP.getTime(); + if (now == null) { + continue; + } + + // Check the total directory size if we haven't in a while + if (this.shouldCalculateDirectorySize(now)) { + this.calculateDirectorySize(now); + } + + Thread.sleep(59000); + } + } catch (InterruptedException e) { + // Fall-through to exit thread... + } + } + + public void shutdown() { + isStopping = true; + this.interrupt(); + instance = null; + } + + /** + * Check if data relating to a transaction is allowed to + * exist on this node, therefore making it a mirror for this data. + * + * @param arbitraryTransactionData - the transaction + * @return boolean - whether to prefetch or not + */ + public boolean canStoreData(ArbitraryTransactionData arbitraryTransactionData) { + String name = arbitraryTransactionData.getName(); + + // We already have RAW_DATA on chain, so we only need to store data associated with hashes + if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) { + return false; + } + + // Don't store data unless it's an allowed type (public/private) + if (!this.isDataTypeAllowed(arbitraryTransactionData)) { + return false; + } + + // Don't check for storage limits here, as it can cause the cleanup manager to delete existing data + + // Check if our storage policy and and lists allow us to host data for this name + switch (Settings.getInstance().getStoragePolicy()) { + case FOLLOWED_OR_VIEWED: + case ALL: + case VIEWED: + // If the policy includes viewed data, we can host it as long as it's not blocked + return !this.isNameBlocked(name); + + case FOLLOWED: + // If the policy is for followed data only, we have to be following it + return this.isFollowingName(name); + + // For NONE or all else, we shouldn't host this data + case NONE: + default: + return false; + } + } + + /** + * Check if data relating to a transaction should be downloaded + * automatically, making this node a mirror for that data. + * + * @param arbitraryTransactionData - the transaction + * @return boolean - whether to prefetch or not + */ + public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { + String name = arbitraryTransactionData.getName(); + + // Only fetch data associated with hashes, as we already have RAW_DATA + if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) { + return false; + } + + // Don't fetch anything more if we're (nearly) out of space + // Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to + // avoid a fetch/delete loop + if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) { + return false; + } + + // Don't fetch anything if we're (nearly) out of space for this name + // Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to + // avoid a fetch/delete loop + if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) { + return false; + } + + // Don't store data unless it's an allowed type (public/private) + if (!this.isDataTypeAllowed(arbitraryTransactionData)) { + return false; + } + + // Handle transactions without names differently + if (name == null) { + return this.shouldPreFetchDataWithoutName(); + } + + // Never fetch data from blocked names, even if they are followed + if (this.isNameBlocked(name)) { + return false; + } + + switch (Settings.getInstance().getStoragePolicy()) { + case FOLLOWED: + case FOLLOWED_OR_VIEWED: + return this.isFollowingName(name); + + case ALL: + return true; + + case NONE: + case VIEWED: + default: + return false; + } + } + + /** + * Don't call this method directly. + * Use the wrapper method shouldPreFetchData() instead, as it contains + * additional checks. + * + * @return boolean - whether the storage policy allows for unnamed data + */ + private boolean shouldPreFetchDataWithoutName() { + switch (Settings.getInstance().getStoragePolicy()) { + case ALL: + return true; + + case NONE: + case VIEWED: + case FOLLOWED: + case FOLLOWED_OR_VIEWED: + default: + return false; + } + } + + private boolean isDataTypeAllowed(ArbitraryTransactionData arbitraryTransactionData) { + byte[] secret = arbitraryTransactionData.getSecret(); + boolean hasSecret = (secret != null && secret.length == 32); + + if (!Settings.getInstance().isPrivateDataEnabled() && !hasSecret) { + // Private data isn't enabled so we can't store data without a valid secret + return false; + } + if (!Settings.getInstance().isPublicDataEnabled() && hasSecret) { + // Public data isn't enabled so we can't store data with a secret + return false; + } + return true; + } + + public boolean isNameBlocked(String name) { + return ResourceListManager.getInstance().listContains("blockedNames", name, false); + } + + private boolean isFollowingName(String name) { + return ResourceListManager.getInstance().listContains("followedNames", name, false); + } + + public List followedNames() { + return ResourceListManager.getInstance().getStringsInList("followedNames"); + } + + private int followedNamesCount() { + return ResourceListManager.getInstance().getItemCountForList("followedNames"); + } + + + public List loadAllHostedTransactions(Repository repository){ + + List arbitraryTransactionDataList = new ArrayList<>(); + + // Find all hosted paths + List allPaths = this.findAllHostedPaths(); + + // Loop through each path and attempt to match it to a signature + for (Path path : allPaths) { + try { + String[] contents = path.toFile().list(); + if (contents == null || contents.length == 0) { + // Ignore empty directories + continue; + } + + String signature58 = path.getFileName().toString(); + byte[] signature = Base58.decode(signature58); + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (transactionData == null || transactionData.getType() != Transaction.TransactionType.ARBITRARY) { + continue; + } + arbitraryTransactionDataList.add((ArbitraryTransactionData) transactionData); + + } catch (DataException e) { + continue; + } + } + + // Sort by newest first + arbitraryTransactionDataList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed()); + + return arbitraryTransactionDataList; + } + // Hosted data + + public List listAllHostedTransactions(Repository repository, Integer limit, Integer offset) { + // Load from cache if we can, to avoid disk reads + + if (this.hostedTransactions != null) { + return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset); + } + + this.hostedTransactions = this.loadAllHostedTransactions(repository); + + return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset); + } + + /** + * searchHostedTransactions + * Allow to run a query against hosted data names and return matches if there are any + * @param repository + * @param query + * @param limit + * @param offset + * @return + */ + + public List searchHostedTransactions(Repository repository, String query, Integer limit, Integer offset) { + // Load from results cache if we can (results that exists for the same query), to avoid disk reads + if (this.searchResultsTransactions != null && this.searchQuery.equals(query.toLowerCase())) { + return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset); + } + + // Using cache if we can, to avoid disk reads + if (this.hostedTransactions == null) { + this.hostedTransactions = this.loadAllHostedTransactions(repository); + } + + this.searchQuery = query.toLowerCase(); //set the searchQuery so that it can be checked on the next call + + List searchResultsList = new ArrayList<>(); + + // Loop through cached hostedTransactions + for (ArbitraryTransactionData atd : this.hostedTransactions) { + try { + if (atd.getName() != null && atd.getName().toLowerCase().contains(this.searchQuery)) { + searchResultsList.add(atd); + } + else if (atd.getIdentifier() != null && atd.getIdentifier().toLowerCase().contains(this.searchQuery)) { + searchResultsList.add(atd); + } + + } catch (Exception e) { + continue; + } + } + + // Sort by newest first + searchResultsList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed()); + + // Update cache + this.searchResultsTransactions = searchResultsList; + + return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset); + } + + /** + * Warning: this method will walk through the entire data directory + * Do not call it too frequently as it could create high disk load + * in environments with a large amount of hosted data. + * @return a list of paths that are being hosted + */ + public List findAllHostedPaths() { + Path dataPath = Paths.get(Settings.getInstance().getDataPath()); + Path tempPath = Paths.get(Settings.getInstance().getTempDataPath()); + + // Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length + // Also exclude the _temp and _misc paths if present + List allPaths = new ArrayList<>(); + try { + allPaths = Files.walk(dataPath, 3) + .filter(Files::isDirectory) + .filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString()) + && !path.toString().contains("_misc") + && path.getFileName().toString().length() > 32) + .collect(Collectors.toList()); + } + catch (IOException | UncheckedIOException e) { + LOGGER.info("Unable to walk through hosted data: {}", e.getMessage()); + } + + return allPaths; + } + + public void invalidateHostedTransactionsCache() { + this.hostedTransactions = null; + } + + + // Size limits + + /** + * Rate limit to reduce IO load + */ + public boolean shouldCalculateDirectorySize(Long now) { + if (now == null) { + return false; + } + // If storage capacity is null, we need to calculate it + if (this.storageCapacity == null) { + return true; + } + // If we haven't checked for a while, we need to check it now + if (now - lastDirectorySizeCheck > DIRECTORY_SIZE_CHECK_INTERVAL) { + return true; + } + + // We shouldn't check this time, as we want to reduce IO load on the SSD/HDD + return false; + } + + public void calculateDirectorySize(Long now) { + if (now == null) { + return; + } + + long totalSize = 0; + long remainingCapacity = 0; + + // Calculate remaining capacity + try { + remainingCapacity = this.getRemainingUsableStorageCapacity(); + } catch (IOException e) { + LOGGER.info("Unable to calculate remaining storage capacity: {}", e.getMessage()); + return; + } + + // Calculate total size of data directory + LOGGER.trace("Calculating data directory size..."); + Path dataDirectoryPath = Paths.get(Settings.getInstance().getDataPath()); + if (dataDirectoryPath.toFile().exists()) { + totalSize += FileUtils.sizeOfDirectory(dataDirectoryPath.toFile()); + } + + // Add total size of temp directory, if it's not already inside the data directory + Path tempDirectoryPath = Paths.get(Settings.getInstance().getTempDataPath()); + if (tempDirectoryPath.toFile().exists()) { + if (!FilesystemUtils.isChild(tempDirectoryPath, dataDirectoryPath)) { + LOGGER.trace("Calculating temp directory size..."); + totalSize += FileUtils.sizeOfDirectory(dataDirectoryPath.toFile()); + } + } + + this.totalDirectorySize = totalSize; + this.lastDirectorySizeCheck = now; + + // It's essential that used space (this.totalDirectorySize) is included in the storage capacity + LOGGER.trace("Calculating total storage capacity..."); + long storageCapacity = remainingCapacity + this.totalDirectorySize; + + // Make sure to limit the storage capacity if the user is overriding it in the settings + if (Settings.getInstance().getMaxStorageCapacity() != null) { + storageCapacity = Math.min(storageCapacity, Settings.getInstance().getMaxStorageCapacity()); + } + this.storageCapacity = storageCapacity; + + LOGGER.info("Total used: {} bytes, Total capacity: {} bytes", this.totalDirectorySize, this.storageCapacity); + } + + private long getRemainingUsableStorageCapacity() throws IOException { + // Create data directory if it doesn't exist so that we can perform calculations on it + Path dataDirectoryPath = Paths.get(Settings.getInstance().getDataPath()); + if (!dataDirectoryPath.toFile().exists()) { + Files.createDirectories(dataDirectoryPath); + } + + return dataDirectoryPath.toFile().getUsableSpace(); + } + + public long getTotalDirectorySize() { + return this.totalDirectorySize; + } + + public boolean isStorageSpaceAvailable(double threshold) { + if (!this.isStorageCapacityCalculated()) { + return false; + } + + long maxStorageCapacity = (long)((double)this.storageCapacity * threshold); + if (this.totalDirectorySize >= maxStorageCapacity) { + return false; + } + return true; + } + + public boolean isStorageSpaceAvailableForName(Repository repository, String name, double threshold) { + if (!this.isStorageSpaceAvailable(threshold)) { + // No storage space available at all, so no need to check this name + return false; + } + + if (name == null) { + // This transaction doesn't have a name, so fall back to total space limitations + return true; + } + + int followedNamesCount = this.followedNamesCount(); + if (followedNamesCount == 0) { + // Not following any names, so we have space + return true; + } + + long totalSizeForName = 0; + long maxStoragePerName = this.storageCapacityPerName(threshold); + + // Fetch all hosted transactions + List hostedTransactions = this.listAllHostedTransactions(repository, null, null); + for (ArbitraryTransactionData transactionData : hostedTransactions) { + String transactionName = transactionData.getName(); + if (!Objects.equals(name, transactionName)) { + // Transaction relates to a different name + continue; + } + + totalSizeForName += transactionData.getSize(); + } + + // Have we reached the limit for this name? + if (totalSizeForName > maxStoragePerName) { + return false; + } + + return true; + } + + public long storageCapacityPerName(double threshold) { + int followedNamesCount = this.followedNamesCount(); + if (followedNamesCount == 0) { + // Not following any names, so we have the total space available + return this.getStorageCapacityIncludingThreshold(threshold); + } + + double maxStorageCapacity = (double)this.storageCapacity * threshold; + long maxStoragePerName = (long)(maxStorageCapacity / (double)followedNamesCount); + + return maxStoragePerName; + } + + public boolean isStorageCapacityCalculated() { + return (this.storageCapacity != null); + } + + public Long getStorageCapacity() { + return this.storageCapacity; + } + + public Long getStorageCapacityIncludingThreshold(double threshold) { + if (this.storageCapacity == null) { + return null; + } + return (long)(this.storageCapacity * threshold); + } +} diff --git a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java new file mode 100644 index 000000000..54fba6991 --- /dev/null +++ b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java @@ -0,0 +1,110 @@ +package org.qortal.controller.repository; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.utils.NTP; + +public class AtStatesPruner implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(AtStatesPruner.class); + + @Override + public void run() { + Thread.currentThread().setName("AT States pruner"); + + boolean archiveMode = false; + if (!Settings.getInstance().isTopOnly()) { + // Top-only mode isn't enabled, but we might want to prune for the purposes of archiving + if (!Settings.getInstance().isArchiveEnabled()) { + // No pruning or archiving, so we must not prune anything + return; + } + else { + // We're allowed to prune blocks that have already been archived + archiveMode = true; + } + } + + try (final Repository repository = RepositoryManager.getRepository()) { + int pruneStartHeight = repository.getATRepository().getAtPruneHeight(); + + repository.discardChanges(); + repository.getATRepository().rebuildLatestAtStates(); + + while (!Controller.isStopping()) { + repository.discardChanges(); + + Thread.sleep(Settings.getInstance().getAtStatesPruneInterval()); + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) + continue; + + // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages + if (Synchronizer.getInstance().isSynchronizing()) + continue; + + // Prune AT states for all blocks up until our latest minus pruneBlockLimit + final int ourLatestHeight = chainTip.getHeight(); + int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); + + // In archive mode we are only allowed to trim blocks that have already been archived + if (archiveMode) { + upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1; + + // TODO: validate that the actual archived data exists before pruning it? + } + + int upperBatchHeight = pruneStartHeight + Settings.getInstance().getAtStatesPruneBatchSize(); + int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); + + if (pruneStartHeight >= upperPruneHeight) + continue; + + LOGGER.debug(String.format("Pruning AT states between blocks %d and %d...", pruneStartHeight, upperPruneHeight)); + + int numAtStatesPruned = repository.getATRepository().pruneAtStates(pruneStartHeight, upperPruneHeight); + repository.saveChanges(); + int numAtStateDataRowsTrimmed = repository.getATRepository().trimAtStates( + pruneStartHeight, upperPruneHeight, Settings.getInstance().getAtStatesTrimLimit()); + repository.saveChanges(); + + if (numAtStatesPruned > 0 || numAtStateDataRowsTrimmed > 0) { + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.debug(() -> String.format("Pruned %d AT state%s between blocks %d and %d", + numAtStatesPruned, (numAtStatesPruned != 1 ? "s" : ""), + finalPruneStartHeight, upperPruneHeight)); + } else { + // Can we move onto next batch? + if (upperPrunableHeight > upperBatchHeight) { + pruneStartHeight = upperBatchHeight; + repository.getATRepository().setAtPruneHeight(pruneStartHeight); + repository.getATRepository().rebuildLatestAtStates(); + repository.saveChanges(); + + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.debug(() -> String.format("Bumping AT state base prune height to %d", finalPruneStartHeight)); + } + else { + // We've pruned up to the upper prunable height + // Back off for a while to save CPU for syncing + repository.discardChanges(); + Thread.sleep(5*60*1000L); + } + } + } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to prune AT states: %s", e.getMessage())); + } catch (InterruptedException e) { + // Time to exit + } + } + +} diff --git a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java new file mode 100644 index 000000000..d3bdc3457 --- /dev/null +++ b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java @@ -0,0 +1,82 @@ +package org.qortal.controller.repository; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.utils.NTP; + +public class AtStatesTrimmer implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(AtStatesTrimmer.class); + + @Override + public void run() { + Thread.currentThread().setName("AT States trimmer"); + + try (final Repository repository = RepositoryManager.getRepository()) { + int trimStartHeight = repository.getATRepository().getAtTrimHeight(); + + repository.discardChanges(); + repository.getATRepository().rebuildLatestAtStates(); + + while (!Controller.isStopping()) { + repository.discardChanges(); + + Thread.sleep(Settings.getInstance().getAtStatesTrimInterval()); + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) + continue; + + // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages + if (Synchronizer.getInstance().isSynchronizing()) + continue; + + long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime(); + // We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks + long chainTrimmableTimestamp = chainTip.getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime(); + + long upperTrimmableTimestamp = Math.min(currentTrimmableTimestamp, chainTrimmableTimestamp); + int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp); + + int upperBatchHeight = trimStartHeight + Settings.getInstance().getAtStatesTrimBatchSize(); + int upperTrimHeight = Math.min(upperBatchHeight, upperTrimmableHeight); + + if (trimStartHeight >= upperTrimHeight) + continue; + + int numAtStatesTrimmed = repository.getATRepository().trimAtStates(trimStartHeight, upperTrimHeight, Settings.getInstance().getAtStatesTrimLimit()); + repository.saveChanges(); + + if (numAtStatesTrimmed > 0) { + final int finalTrimStartHeight = trimStartHeight; + LOGGER.debug(() -> String.format("Trimmed %d AT state%s between blocks %d and %d", + numAtStatesTrimmed, (numAtStatesTrimmed != 1 ? "s" : ""), + finalTrimStartHeight, upperTrimHeight)); + } else { + // Can we move onto next batch? + if (upperTrimmableHeight > upperBatchHeight) { + trimStartHeight = upperBatchHeight; + repository.getATRepository().setAtTrimHeight(trimStartHeight); + repository.getATRepository().rebuildLatestAtStates(); + repository.saveChanges(); + + final int finalTrimStartHeight = trimStartHeight; + LOGGER.debug(() -> String.format("Bumping AT state base trim height to %d", finalTrimStartHeight)); + } + } + } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to trim AT states: %s", e.getMessage())); + } catch (InterruptedException e) { + // Time to exit + } + } + +} diff --git a/src/main/java/org/qortal/controller/repository/BlockArchiver.java b/src/main/java/org/qortal/controller/repository/BlockArchiver.java new file mode 100644 index 000000000..ef26610c1 --- /dev/null +++ b/src/main/java/org/qortal/controller/repository/BlockArchiver.java @@ -0,0 +1,114 @@ +package org.qortal.controller.repository; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; +import org.qortal.data.block.BlockData; +import org.qortal.repository.*; +import org.qortal.settings.Settings; +import org.qortal.transform.TransformationException; +import org.qortal.utils.NTP; + +import java.io.IOException; + +public class BlockArchiver implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(BlockArchiver.class); + + private static final long INITIAL_SLEEP_PERIOD = 0L; // TODO: 5 * 60 * 1000L + 1234L; // ms + + public void run() { + Thread.currentThread().setName("Block archiver"); + + if (!Settings.getInstance().isArchiveEnabled()) { + return; + } + + try (final Repository repository = RepositoryManager.getRepository()) { + // Don't even start building until initial rush has ended + Thread.sleep(INITIAL_SLEEP_PERIOD); + + int startHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); + + // Don't attempt to archive if we have no ATStatesHeightIndex, as it will be too slow + boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex(); + if (!hasAtStatesHeightIndex) { + LOGGER.info("Unable to start block archiver due to missing ATStatesHeightIndex. Bootstrapping is recommended."); + repository.discardChanges(); + return; + } + + LOGGER.info("Starting block archiver from height {}...", startHeight); + + while (!Controller.isStopping()) { + repository.discardChanges(); + + Thread.sleep(Settings.getInstance().getArchiveInterval()); + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) { + continue; + } + + // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages + if (Synchronizer.getInstance().isSynchronizing()) { + continue; + } + + // Don't attempt to archive if we're not synced yet + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) { + continue; + } + + + // Build cache of blocks + try { + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository); + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + switch (result) { + case OK: + // Increment block archive height + startHeight += writer.getWrittenCount(); + repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight); + repository.saveChanges(); + break; + + case STOPPING: + return; + + // We've reached the limit of the blocks we can archive + // Sleep for a while to allow more to become available + case NOT_ENOUGH_BLOCKS: + // We didn't reach our file size target, so that must mean that we don't have enough blocks + // yet or something went wrong. Sleep for a while and then try again. + repository.discardChanges(); + Thread.sleep(60 * 60 * 1000L); // 1 hour + break; + + case BLOCK_NOT_FOUND: + // We tried to archive a block that didn't exist. This is a major failure and likely means + // that a bootstrap or re-sync is needed. Try again every minute until then. + LOGGER.info("Error: block not found when building archive. If this error persists, " + + "a bootstrap or re-sync may be needed."); + repository.discardChanges(); + Thread.sleep( 60 * 1000L); // 1 minute + break; + } + + } catch (IOException | TransformationException e) { + LOGGER.info("Caught exception when creating block cache", e); + } + + } + } catch (DataException e) { + LOGGER.info("Caught exception when creating block cache", e); + } catch (InterruptedException e) { + // Do nothing + } + + } + +} diff --git a/src/main/java/org/qortal/controller/repository/BlockPruner.java b/src/main/java/org/qortal/controller/repository/BlockPruner.java new file mode 100644 index 000000000..03fb38b94 --- /dev/null +++ b/src/main/java/org/qortal/controller/repository/BlockPruner.java @@ -0,0 +1,115 @@ +package org.qortal.controller.repository; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.utils.NTP; + +public class BlockPruner implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(BlockPruner.class); + + @Override + public void run() { + Thread.currentThread().setName("Block pruner"); + + boolean archiveMode = false; + if (!Settings.getInstance().isTopOnly()) { + // Top-only mode isn't enabled, but we might want to prune for the purposes of archiving + if (!Settings.getInstance().isArchiveEnabled()) { + // No pruning or archiving, so we must not prune anything + return; + } + else { + // We're allowed to prune blocks that have already been archived + archiveMode = true; + } + } + + try (final Repository repository = RepositoryManager.getRepository()) { + int pruneStartHeight = repository.getBlockRepository().getBlockPruneHeight(); + + // Don't attempt to prune if we have no ATStatesHeightIndex, as it will be too slow + boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex(); + if (!hasAtStatesHeightIndex) { + LOGGER.info("Unable to start block pruner due to missing ATStatesHeightIndex. Bootstrapping is recommended."); + return; + } + + while (!Controller.isStopping()) { + repository.discardChanges(); + + Thread.sleep(Settings.getInstance().getBlockPruneInterval()); + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) + continue; + + // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages + if (Synchronizer.getInstance().isSynchronizing()) { + continue; + } + + // Don't attempt to prune if we're not synced yet + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) { + continue; + } + + // Prune all blocks up until our latest minus pruneBlockLimit + final int ourLatestHeight = chainTip.getHeight(); + int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); + + // In archive mode we are only allowed to trim blocks that have already been archived + if (archiveMode) { + upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1; + } + + int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize(); + int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); + + if (pruneStartHeight >= upperPruneHeight) { + continue; + } + + LOGGER.debug(String.format("Pruning blocks between %d and %d...", pruneStartHeight, upperPruneHeight)); + + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(pruneStartHeight, upperPruneHeight); + repository.saveChanges(); + + if (numBlocksPruned > 0) { + LOGGER.debug(String.format("Pruned %d block%s between %d and %d", + numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""), + pruneStartHeight, upperPruneHeight)); + } else { + final int nextPruneHeight = upperPruneHeight + 1; + repository.getBlockRepository().setBlockPruneHeight(nextPruneHeight); + repository.saveChanges(); + LOGGER.debug(String.format("Bumping block base prune height to %d", pruneStartHeight)); + + // Can we move onto next batch? + if (upperPrunableHeight > nextPruneHeight) { + pruneStartHeight = nextPruneHeight; + } + else { + // We've pruned up to the upper prunable height + // Back off for a while to save CPU for syncing + repository.discardChanges(); + Thread.sleep(10*60*1000L); + } + } + } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to prune blocks: %s", e.getMessage())); + } catch (InterruptedException e) { + // Time to exit + } + } + +} diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java new file mode 100644 index 000000000..5e54b9056 --- /dev/null +++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java @@ -0,0 +1,372 @@ +package org.qortal.controller.repository; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; +import org.qortal.data.naming.NameData; +import org.qortal.data.transaction.*; +import org.qortal.naming.Name; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.utils.Unicode; + +import java.util.*; + +public class NamesDatabaseIntegrityCheck { + + private static final Logger LOGGER = LogManager.getLogger(NamesDatabaseIntegrityCheck.class); + + private static final List ALL_NAME_TX_TYPE = Arrays.asList( + TransactionType.REGISTER_NAME, + TransactionType.UPDATE_NAME, + TransactionType.BUY_NAME, + TransactionType.SELL_NAME + ); + + private List nameTransactions = new ArrayList<>(); + + public int rebuildName(String name, Repository repository) { + int modificationCount = 0; + try { + List transactions = this.fetchAllTransactionsInvolvingName(name, repository); + if (transactions.isEmpty()) { + // This name was never registered, so there's nothing to do + return modificationCount; + } + + // Loop through each past transaction and re-apply it to the Names table + for (TransactionData currentTransaction : transactions) { + + // Process REGISTER_NAME transactions + if (currentTransaction.getType() == TransactionType.REGISTER_NAME) { + RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) currentTransaction; + Name nameObj = new Name(repository, registerNameTransactionData); + nameObj.register(); + modificationCount++; + LOGGER.trace("Processed REGISTER_NAME transaction for name {}", name); + } + + // Process UPDATE_NAME transactions + if (currentTransaction.getType() == TransactionType.UPDATE_NAME) { + UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) currentTransaction; + + if (Objects.equals(updateNameTransactionData.getNewName(), name) && + !Objects.equals(updateNameTransactionData.getName(), updateNameTransactionData.getNewName())) { + // This renames an existing name, so we need to process that instead + this.rebuildName(updateNameTransactionData.getName(), repository); + } + else { + Name nameObj = new Name(repository, name); + if (nameObj != null && nameObj.getNameData() != null) { + nameObj.update(updateNameTransactionData); + modificationCount++; + LOGGER.trace("Processed UPDATE_NAME transaction for name {}", name); + } else { + // Something went wrong + throw new DataException(String.format("Name data not found for name %s", updateNameTransactionData.getName())); + } + } + } + + // Process SELL_NAME transactions + if (currentTransaction.getType() == TransactionType.SELL_NAME) { + SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) currentTransaction; + Name nameObj = new Name(repository, sellNameTransactionData.getName()); + if (nameObj != null && nameObj.getNameData() != null) { + nameObj.sell(sellNameTransactionData); + modificationCount++; + LOGGER.trace("Processed SELL_NAME transaction for name {}", name); + } + else { + // Something went wrong + throw new DataException(String.format("Name data not found for name %s", sellNameTransactionData.getName())); + } + } + + // Process BUY_NAME transactions + if (currentTransaction.getType() == TransactionType.BUY_NAME) { + BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) currentTransaction; + Name nameObj = new Name(repository, buyNameTransactionData.getName()); + if (nameObj != null && nameObj.getNameData() != null) { + nameObj.buy(buyNameTransactionData); + modificationCount++; + LOGGER.trace("Processed BUY_NAME transaction for name {}", name); + } + else { + // Something went wrong + throw new DataException(String.format("Name data not found for name %s", buyNameTransactionData.getName())); + } + } + } + + } catch (DataException e) { + LOGGER.info("Unable to run integrity check for name {}: {}", name, e.getMessage()); + } + + return modificationCount; + } + + public int rebuildAllNames() { + int modificationCount = 0; + try (final Repository repository = RepositoryManager.getRepository()) { + List names = this.fetchAllNames(repository); + for (String name : names) { + modificationCount += this.rebuildName(name, repository); + } + repository.saveChanges(); + } + catch (DataException e) { + LOGGER.info("Error when running integrity check for all names: {}", e.getMessage()); + } + + //LOGGER.info("modificationCount: {}", modificationCount); + return modificationCount; + } + + public void runIntegrityCheck() { + boolean integrityCheckFailed = false; + try (final Repository repository = RepositoryManager.getRepository()) { + + // Fetch all the (confirmed) REGISTER_NAME transactions + List registerNameTransactions = this.fetchRegisterNameTransactions(); + + // Loop through each REGISTER_NAME txn signature and request the full transaction data + for (RegisterNameTransactionData registerNameTransactionData : registerNameTransactions) { + String registeredName = registerNameTransactionData.getName(); + NameData nameData = repository.getNameRepository().fromName(registeredName); + + // Check to see if this name has been updated or bought at any point + TransactionData latestUpdate = this.fetchLatestModificationTransactionInvolvingName(registeredName, repository); + if (latestUpdate == null) { + // Name was never updated once registered + // We expect this name to still be registered to this transaction's creator + + if (nameData == null) { + LOGGER.info("Error: registered name {} doesn't exist in Names table. Adding...", registeredName); + integrityCheckFailed = true; + } + else { + LOGGER.trace("Registered name {} is correctly registered", registeredName); + } + + // Check the owner is correct + PublicKeyAccount creator = new PublicKeyAccount(repository, registerNameTransactionData.getCreatorPublicKey()); + if (!Objects.equals(creator.getAddress(), nameData.getOwner())) { + LOGGER.info("Error: registered name {} is owned by {}, but it should be {}", + registeredName, nameData.getOwner(), creator.getAddress()); + integrityCheckFailed = true; + } + else { + LOGGER.trace("Registered name {} has the correct owner", registeredName); + } + } + else { + // Check if owner is correct after update + + // Check for name updates + if (latestUpdate.getType() == TransactionType.UPDATE_NAME) { + UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) latestUpdate; + PublicKeyAccount creator = new PublicKeyAccount(repository, updateNameTransactionData.getCreatorPublicKey()); + + // When this name is the "new name", we expect the current owner to match the txn creator + if (Objects.equals(updateNameTransactionData.getNewName(), registeredName)) { + if (!Objects.equals(creator.getAddress(), nameData.getOwner())) { + LOGGER.info("Error: registered name {} is owned by {}, but it should be {}", + registeredName, nameData.getOwner(), creator.getAddress()); + integrityCheckFailed = true; + } + else { + LOGGER.trace("Registered name {} has the correct owner after being updated", registeredName); + } + } + + // When this name is the old name, we expect the "new name"'s owner to match the txn creator + // The old name will then be unregistered, or re-registered. + // FUTURE: check database integrity for names that have been updated and then the original name re-registered + else if (Objects.equals(updateNameTransactionData.getName(), registeredName)) { + String newName = updateNameTransactionData.getNewName(); + if (newName == null || newName.length() == 0) { + // If new name is blank (or maybe null, just to be safe), it means that it stayed the same + newName = registeredName; + } + NameData newNameData = repository.getNameRepository().fromName(newName); + if (!Objects.equals(creator.getAddress(), newNameData.getOwner())) { + LOGGER.info("Error: registered name {} is owned by {}, but it should be {}", + updateNameTransactionData.getNewName(), newNameData.getOwner(), creator.getAddress()); + integrityCheckFailed = true; + } + else { + LOGGER.trace("Registered name {} has the correct owner after being updated", updateNameTransactionData.getNewName()); + } + } + + else { + LOGGER.info("Unhandled update case for name {}", registeredName); + } + } + + // Check for name buys + else if (latestUpdate.getType() == TransactionType.BUY_NAME) { + BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) latestUpdate; + PublicKeyAccount creator = new PublicKeyAccount(repository, buyNameTransactionData.getCreatorPublicKey()); + if (!Objects.equals(creator.getAddress(), nameData.getOwner())) { + LOGGER.info("Error: registered name {} is owned by {}, but it should be {}", + registeredName, nameData.getOwner(), creator.getAddress()); + integrityCheckFailed = true; + } + else { + LOGGER.trace("Registered name {} has the correct owner after being bought", registeredName); + } + } + + // Check for name sells + else if (latestUpdate.getType() == TransactionType.SELL_NAME) { + SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) latestUpdate; + PublicKeyAccount creator = new PublicKeyAccount(repository, sellNameTransactionData.getCreatorPublicKey()); + if (!Objects.equals(creator.getAddress(), nameData.getOwner())) { + LOGGER.info("Error: registered name {} is owned by {}, but it should be {}", + registeredName, nameData.getOwner(), creator.getAddress()); + integrityCheckFailed = true; + } + else { + LOGGER.trace("Registered name {} has the correct owner after being listed for sale", registeredName); + } + } + + else { + LOGGER.info("Unhandled case for name {}", registeredName); + } + + } + + } + + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to trim online accounts signatures: %s", e.getMessage())); + integrityCheckFailed = true; + } + + if (integrityCheckFailed) { + LOGGER.info("Registered names database integrity check failed. Bootstrapping is recommended."); + } else { + LOGGER.info("Registered names database integrity check passed."); + } + } + + private List fetchRegisterNameTransactions() { + List registerNameTransactions = new ArrayList<>(); + + for (TransactionData transactionData : this.nameTransactions) { + if (transactionData.getType() == TransactionType.REGISTER_NAME) { + RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData; + registerNameTransactions.add(registerNameTransactionData); + } + } + return registerNameTransactions; + } + + private void fetchAllNameTransactions(Repository repository) throws DataException { + List nameTransactions = new ArrayList<>(); + + // Fetch all the confirmed REGISTER_NAME transaction signatures + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria( + null, null, null, ALL_NAME_TX_TYPE, null, null, + null, ConfirmationStatus.CONFIRMED, null, null, false); + + for (byte[] signature : signatures) { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + nameTransactions.add(transactionData); + } + this.nameTransactions = nameTransactions; + } + + public List fetchAllTransactionsInvolvingName(String name, Repository repository) throws DataException { + List signatures = new ArrayList<>(); + String reducedName = Unicode.sanitize(name); + + List registerNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria( + TransactionType.REGISTER_NAME, Arrays.asList("(name = ? OR reduced_name = ?)"), Arrays.asList(name, reducedName)); + signatures.addAll(registerNameTransactions); + + List updateNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria( + TransactionType.UPDATE_NAME, + Arrays.asList("(name = ? OR new_name = ? OR (reduced_new_name != '' AND reduced_new_name = ?))"), + Arrays.asList(name, name, reducedName)); + signatures.addAll(updateNameTransactions); + + List sellNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria( + TransactionType.SELL_NAME, Arrays.asList("name = ?"), Arrays.asList(name)); + signatures.addAll(sellNameTransactions); + + List buyNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria( + TransactionType.BUY_NAME, Arrays.asList("name = ?"), Arrays.asList(name)); + signatures.addAll(buyNameTransactions); + + List transactions = new ArrayList<>(); + for (byte[] signature : signatures) { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + // Filter out any unconfirmed transactions + if (transactionData.getBlockHeight() != null && transactionData.getBlockHeight() > 0) { + transactions.add(transactionData); + } + } + return transactions; + } + + private TransactionData fetchLatestModificationTransactionInvolvingName(String registeredName, Repository repository) throws DataException { + List transactionsInvolvingName = this.fetchAllTransactionsInvolvingName(registeredName, repository); + + // Get the latest update for this name (excluding REGISTER_NAME transactions) + TransactionData latestUpdateToName = transactionsInvolvingName.stream() + .filter(txn -> txn.getType() != TransactionType.REGISTER_NAME) + .max(Comparator.comparing(TransactionData::getTimestamp)) + .orElse(null); + + return latestUpdateToName; + } + + private List fetchAllNames(Repository repository) throws DataException { + List names = new ArrayList<>(); + + // Fetch all the confirmed name transactions + if (this.nameTransactions.isEmpty()) { + this.fetchAllNameTransactions(repository); + } + + for (TransactionData transactionData : this.nameTransactions) { + + if ((transactionData instanceof RegisterNameTransactionData)) { + RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData; + if (!names.contains(registerNameTransactionData.getName())) { + names.add(registerNameTransactionData.getName()); + } + } + if ((transactionData instanceof UpdateNameTransactionData)) { + UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData; + if (!names.contains(updateNameTransactionData.getName())) { + names.add(updateNameTransactionData.getName()); + } + if (!names.contains(updateNameTransactionData.getNewName())) { + names.add(updateNameTransactionData.getNewName()); + } + } + if ((transactionData instanceof BuyNameTransactionData)) { + BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData; + if (!names.contains(buyNameTransactionData.getName())) { + names.add(buyNameTransactionData.getName()); + } + } + if ((transactionData instanceof SellNameTransactionData)) { + SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData; + if (!names.contains(sellNameTransactionData.getName())) { + names.add(sellNameTransactionData.getName()); + } + } + } + return names; + } + +} diff --git a/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java b/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java new file mode 100644 index 000000000..dfd9d45eb --- /dev/null +++ b/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java @@ -0,0 +1,81 @@ +package org.qortal.controller.repository; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.block.BlockChain; +import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.utils.NTP; + +public class OnlineAccountsSignaturesTrimmer implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(OnlineAccountsSignaturesTrimmer.class); + + private static final long INITIAL_SLEEP_PERIOD = 5 * 60 * 1000L + 1234L; // ms + + public void run() { + Thread.currentThread().setName("Online Accounts trimmer"); + + try (final Repository repository = RepositoryManager.getRepository()) { + // Don't even start trimming until initial rush has ended + Thread.sleep(INITIAL_SLEEP_PERIOD); + + int trimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight(); + + while (!Controller.isStopping()) { + repository.discardChanges(); + + Thread.sleep(Settings.getInstance().getOnlineSignaturesTrimInterval()); + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) + continue; + + // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages + if (Synchronizer.getInstance().isSynchronizing()) + continue; + + // Trim blockchain by removing 'old' online accounts signatures + long upperTrimmableTimestamp = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime(); + int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp); + + int upperBatchHeight = trimStartHeight + Settings.getInstance().getOnlineSignaturesTrimBatchSize(); + int upperTrimHeight = Math.min(upperBatchHeight, upperTrimmableHeight); + + if (trimStartHeight >= upperTrimHeight) + continue; + + int numSigsTrimmed = repository.getBlockRepository().trimOldOnlineAccountsSignatures(trimStartHeight, upperTrimHeight); + repository.saveChanges(); + + if (numSigsTrimmed > 0) { + final int finalTrimStartHeight = trimStartHeight; + LOGGER.debug(() -> String.format("Trimmed %d online accounts signature%s between blocks %d and %d", + numSigsTrimmed, (numSigsTrimmed != 1 ? "s" : ""), + finalTrimStartHeight, upperTrimHeight)); + } else { + // Can we move onto next batch? + if (upperTrimmableHeight > upperBatchHeight) { + trimStartHeight = upperBatchHeight; + + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(trimStartHeight); + repository.saveChanges(); + + final int finalTrimStartHeight = trimStartHeight; + LOGGER.debug(() -> String.format("Bumping online accounts signatures base trim height to %d", finalTrimStartHeight)); + } + } + } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to trim online accounts signatures: %s", e.getMessage())); + } catch (InterruptedException e) { + // Time to exit + } + } + +} diff --git a/src/main/java/org/qortal/controller/repository/PruneManager.java b/src/main/java/org/qortal/controller/repository/PruneManager.java new file mode 100644 index 000000000..ec27456fa --- /dev/null +++ b/src/main/java/org/qortal/controller/repository/PruneManager.java @@ -0,0 +1,160 @@ +package org.qortal.controller.repository; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; + +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.settings.Settings; +import org.qortal.utils.DaemonThreadFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +public class PruneManager { + + private static final Logger LOGGER = LogManager.getLogger(PruneManager.class); + + private static PruneManager instance; + + private boolean isTopOnly = Settings.getInstance().isTopOnly(); + private int pruneBlockLimit = Settings.getInstance().getPruneBlockLimit(); + + private ExecutorService executorService; + + private PruneManager() { + + } + + public static synchronized PruneManager getInstance() { + if (instance == null) + instance = new PruneManager(); + + return instance; + } + + public void start() { + this.executorService = Executors.newCachedThreadPool(new DaemonThreadFactory()); + + if (Settings.getInstance().isTopOnly()) { + // Top-only-sync + this.startTopOnlySyncMode(); + } + else if (Settings.getInstance().isArchiveEnabled()) { + // Full node with block archive + this.startFullNodeWithBlockArchive(); + } + else { + // Full node with full SQL support + this.startFullSQLNode(); + } + } + + /** + * Top-only-sync + * In this mode, we delete (prune) all blocks except + * a small number of recent ones. There is no need for + * trimming or archiving, because all relevant blocks + * are deleted. + */ + private void startTopOnlySyncMode() { + this.startPruning(); + + // We don't need the block archive in top-only mode + this.deleteArchive(); + } + + /** + * Full node with block archive + * In this mode we archive trimmed blocks, and then + * prune archived blocks to keep the database small + */ + private void startFullNodeWithBlockArchive() { + this.startTrimming(); + this.startArchiving(); + this.startPruning(); + } + + /** + * Full node with full SQL support + * In this mode we trim the database but don't prune + * or archive any data, because we want to maintain + * full SQL support of old blocks. This mode will not + * be actively maintained but can be used by those who + * need to perform SQL analysis on older blocks. + */ + private void startFullSQLNode() { + this.startTrimming(); + } + + + private void startPruning() { + this.executorService.execute(new AtStatesPruner()); + this.executorService.execute(new BlockPruner()); + } + + private void startTrimming() { + this.executorService.execute(new AtStatesTrimmer()); + this.executorService.execute(new OnlineAccountsSignaturesTrimmer()); + } + + private void startArchiving() { + this.executorService.execute(new BlockArchiver()); + } + + private void deleteArchive() { + if (!Settings.getInstance().isTopOnly()) { + LOGGER.error("Refusing to delete archive when not in top-only mode"); + } + + try { + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive"); + if (archivePath.toFile().exists()) { + LOGGER.info("Deleting block archive because we are in top-only mode..."); + FileUtils.deleteDirectory(archivePath.toFile()); + } + + } catch (IOException e) { + LOGGER.info("Couldn't delete archive: {}", e.getMessage()); + } + } + + public void stop() { + this.executorService.shutdownNow(); + + try { + this.executorService.awaitTermination(2L, TimeUnit.SECONDS); + } catch (InterruptedException e) { + // We tried... + } + } + + public boolean isBlockPruned(int height) throws DataException { + if (!this.isTopOnly) { + return false; + } + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null) { + throw new DataException("Unable to determine chain tip when checking if a block is pruned"); + } + + if (height == 1) { + // We don't prune the genesis block + return false; + } + + final int ourLatestHeight = chainTip.getHeight(); + final int latestUnprunedHeight = ourLatestHeight - this.pruneBlockLimit; + + return (height < latestUnprunedHeight); + } + +} diff --git a/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java b/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java new file mode 100644 index 000000000..84a0d4849 --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java @@ -0,0 +1,30 @@ +package org.qortal.controller.tradebot; + +import java.util.List; + +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; + +public interface AcctTradeBot { + + public enum ResponseResult { OK, BALANCE_ISSUE, NETWORK_ISSUE, TRADE_ALREADY_EXISTS } + + /** Returns list of state names for trade-bot entries that have ended, e.g. redeemed, refunded or cancelled. */ + public List getEndStates(); + + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException; + + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, + CrossChainTradeData crossChainTradeData, String foreignKey, String receivingAddress) throws DataException; + + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException; + + public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException; + +} diff --git a/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java new file mode 100644 index 000000000..038ecdedb --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java @@ -0,0 +1,1274 @@ +package org.qortal.controller.tradebot; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.Address; +import org.bitcoinj.core.AddressFormatException; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.ECKey; +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.TransactionOutput; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.asset.Asset; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.AcctMode; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.BitcoinACCTv1; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.DeployAtTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +/** + * Performing cross-chain trading steps on behalf of user. + *

+ * We deal with three different independent state-spaces here: + *

    + *
  • Qortal blockchain
    • + *
  • Foreign blockchain
    • + *
  • Trade-bot entries
    • + *
+ */ +public class BitcoinACCTv1TradeBot implements AcctTradeBot { + + private static final Logger LOGGER = LogManager.getLogger(BitcoinACCTv1TradeBot.class); + + public enum State implements TradeBot.StateNameAndValueSupplier { + BOB_WAITING_FOR_AT_CONFIRM(10, false, false), + BOB_WAITING_FOR_MESSAGE(15, true, true), + BOB_WAITING_FOR_P2SH_B(20, true, true), + BOB_WAITING_FOR_AT_REDEEM(25, true, true), + BOB_DONE(30, false, false), + BOB_REFUNDED(35, false, false), + + ALICE_WAITING_FOR_P2SH_A(80, true, true), + ALICE_WAITING_FOR_AT_LOCK(85, true, true), + ALICE_WATCH_P2SH_B(90, true, true), + ALICE_DONE(95, false, false), + ALICE_REFUNDING_B(100, true, true), + ALICE_REFUNDING_A(105, true, true), + ALICE_REFUNDED(110, false, false); + + private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); + + public final int value; + public final boolean requiresAtData; + public final boolean requiresTradeData; + + State(int value, boolean requiresAtData, boolean requiresTradeData) { + this.value = value; + this.requiresAtData = requiresAtData; + this.requiresTradeData = requiresTradeData; + } + + public static State valueOf(int value) { + return map.get(value); + } + + @Override + public String getState() { + return this.name(); + } + + @Override + public int getStateValue() { + return this.value; + } + } + + /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ + private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms + + /** P2SH-B output amount to avoid dust threshold (3000 sats/kB). */ + private static final long P2SH_B_OUTPUT_AMOUNT = 1000L; + + private static BitcoinACCTv1TradeBot instance; + + private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDING_B, State.ALICE_REFUNDED).stream() + .map(State::name) + .collect(Collectors.toUnmodifiableList()); + + private BitcoinACCTv1TradeBot() { + } + + public static synchronized BitcoinACCTv1TradeBot getInstance() { + if (instance == null) + instance = new BitcoinACCTv1TradeBot(); + + return instance; + } + + @Override + public List getEndStates() { + return this.endStates; + } + + /** + * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for BTC. + *

+ * Generates: + *

    + *
  • new 'trade' private key
    • + *
  • secret-B
    • + *
+ * Derives: + *
    + *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • + *
  • 'foreign' (as in Bitcoin) public key, public key hash
    • + *
  • HASH160 of secret-B
    • + *
+ * A Qortal AT is then constructed including the following as constants in the 'data segment': + *
    + *
  • 'native'/Qortal 'trade' address - used as a MESSAGE contact
    • + *
  • 'foreign'/Bitcoin public key hash - used by Alice's P2SH scripts to allow redeem
    • + *
  • HASH160 of secret-B - used by AT and P2SH to validate a potential secret-B
    • + *
  • QORT amount on offer by Bob
    • + *
  • BTC amount expected in return by Bob (from Alice)
    • + *
  • trading timeout, in case things go wrong and everyone needs to refund
    • + *
+ * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. + *

+ * Trade-bot will wait for Bob's AT to be deployed before taking next step. + *

+ * @param repository + * @param tradeBotCreateRequest + * @return raw, unsigned DEPLOY_AT transaction + * @throws DataException + */ + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + byte[] secretB = TradeBot.generateSecret(); + byte[] hashOfSecretB = Crypto.hash160(secretB); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + + // Convert Bitcoin receiving address into public key hash (we only support P2PKH at this time) + Address bitcoinReceivingAddress; + try { + bitcoinReceivingAddress = Address.fromString(Bitcoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); + } catch (AddressFormatException e) { + throw new DataException("Unsupported Bitcoin receiving address: " + tradeBotCreateRequest.receivingAddress); + } + if (bitcoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) + throw new DataException("Unsupported Bitcoin receiving address: " + tradeBotCreateRequest.receivingAddress); + + byte[] bitcoinReceivingAccountInfo = bitcoinReceivingAddress.getHash(); + + PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); + + // Deploy AT + long timestamp = NTP.getTime(); + byte[] reference = creator.getLastReference(); + long fee = 0L; + byte[] signature = null; + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); + + String name = "QORT/BTC ACCT"; + String description = "QORT/BTC cross-chain trade"; + String aTType = "ACCT"; + String tags = "ACCT QORT BTC"; + byte[] creationBytes = BitcoinACCTv1.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, hashOfSecretB, tradeBotCreateRequest.qortAmount, + tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); + long amount = tradeBotCreateRequest.fundingQortAmount; + + DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + DeployAtTransaction.ensureATAddress(deployAtTransactionData); + String atAddress = deployAtTransactionData.getAtAddress(); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, BitcoinACCTv1.NAME, + State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, + creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secretB, hashOfSecretB, + SupportedBlockchain.BITCOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + tradeBotCreateRequest.foreignAmount, null, null, null, bitcoinReceivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); + + // Return to user for signing and broadcast as we don't have their Qortal private key + try { + return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); + } catch (TransformationException e) { + throw new DataException("Failed to transform DEPLOY_AT transaction?", e); + } + } + + /** + * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching BTC to an existing offer. + *

+ * Requires a chosen trade offer from Bob, passed by crossChainTradeData + * and access to a Bitcoin wallet via xprv58. + *

+ * The crossChainTradeData contains the current trade offer state + * as extracted from the AT's data segment. + *

+ * Access to a funded wallet is via a Bitcoin BIP32 hierarchical deterministic key, + * passed via xprv58. + * This key will be stored in your node's database + * to allow trade-bot to create/fund the necessary P2SH transactions! + * However, due to the nature of BIP32 keys, it is possible to give the trade-bot + * only a subset of wallet access (see BIP32 for more details). + *

+ * As an example, the xprv58 can be extract from a legacy, password-less + * Electrum wallet by going to the console tab and entering:
+ * wallet.keystore.xprv
+ * which should result in a base58 string starting with either 'xprv' (for Bitcoin main-net) + * or 'tprv' for (Bitcoin test-net). + *

+ * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. + *

+ * If sufficient funds are available, this method will actually fund the P2SH-A + * with the Bitcoin amount expected by 'Bob'. + *

+ * If the Bitcoin transaction is successfully broadcast to the network then the trade-bot entry + * is saved to the repository and the cross-chain trading process commences. + *

+ * Trade-bot will wait for P2SH-A to confirm before taking next step. + *

+ * @param repository + * @param crossChainTradeData chosen trade OFFER that Alice wants to match + * @param xprv58 funded wallet xprv in base58 + * @return true if P2SH-A funding transaction successfully broadcast to Bitcoin network, false otherwise + * @throws DataException + */ + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + byte[] secretA = TradeBot.generateSecret(); + byte[] hashOfSecretA = Crypto.hash160(secretA); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH + + // We need to generate lockTime-A: add tradeTimeout to now + long now = NTP.getTime(); + int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, BitcoinACCTv1.NAME, + State.ALICE_WAITING_FOR_P2SH_A.name(), State.ALICE_WAITING_FOR_P2SH_A.value, + receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secretA, hashOfSecretA, + SupportedBlockchain.BITCOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); + + // Check we have enough funds via xprv58 to fund both P2SHs to cover expectedBitcoin + String tradeForeignAddress = Bitcoin.getInstance().pkhToAddress(tradeForeignPublicKeyHash); + + long p2shFee; + try { + p2shFee = Bitcoin.getInstance().getP2shFee(now); + } catch (ForeignBlockchainException e) { + LOGGER.debug("Couldn't estimate Bitcoin fees?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long fundsRequiredForP2shA = p2shFee /*funding P2SH-A*/ + crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFee /*redeeming/refunding P2SH-A*/; + long fundsRequiredForP2shB = p2shFee /*funding P2SH-B*/ + P2SH_B_OUTPUT_AMOUNT + p2shFee /*redeeming/refunding P2SH-B*/; + long totalFundsRequired = fundsRequiredForP2shA + fundsRequiredForP2shB; + + // As buildSpend also adds a fee, this is more pessimistic than required + Transaction fundingCheckTransaction = Bitcoin.getInstance().buildSpend(xprv58, tradeForeignAddress, totalFundsRequired); + if (fundingCheckTransaction == null) + return ResponseResult.BALANCE_ISSUE; + + // P2SH-A to be funded + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); + String p2shAddress = Bitcoin.getInstance().deriveP2shAddress(redeemScriptBytes); + + // Fund P2SH-A + + // Do not include fee for funding transaction as this is covered by buildSpend() + long amountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFee /*redeeming/refunding P2SH-A*/; + + Transaction p2shFundingTransaction = Bitcoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); + if (p2shFundingTransaction == null) { + LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); + return ResponseResult.BALANCE_ISSUE; + } + + try { + Bitcoin.getInstance().broadcastTransaction(p2shFundingTransaction); + } catch (ForeignBlockchainException e) { + // We couldn't fund P2SH-A at this time + LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); + return ResponseResult.NETWORK_ISSUE; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Waiting for confirmation", p2shAddress)); + + return ResponseResult.OK; + } + + @Override + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) + return true; + + // If the AT doesn't exist then we might as well let the user tidy up + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) + return true; + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + case ALICE_DONE: + case BOB_DONE: + case ALICE_REFUNDED: + case BOB_REFUNDED: + case ALICE_REFUNDING_A: + return true; + + default: + return false; + } + } + + @Override + public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) { + LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); + return; + } + + ATData atData = null; + CrossChainTradeData tradeData = null; + + if (tradeBotState.requiresAtData) { + // Attempt to fetch AT data + atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); + if (atData == null) { + LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + return; + } + + if (tradeBotState.requiresTradeData) { + tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + if (tradeData == null) { + LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); + return; + } + } + } + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + handleBobWaitingForAtConfirm(repository, tradeBotData); + break; + + case ALICE_WAITING_FOR_P2SH_A: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWaitingForP2shA(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_MESSAGE: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_WAITING_FOR_AT_LOCK: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_P2SH_B: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForP2shB(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_WATCH_P2SH_B: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWatchingP2shB(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_AT_REDEEM: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_DONE: + case BOB_DONE: + break; + + case ALICE_REFUNDING_B: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceRefundingP2shB(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_REFUNDING_A: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_REFUNDED: + case BOB_REFUNDED: + break; + } + } + + /** + * Trade-bot is waiting for Bob's AT to deploy. + *

+ * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. + */ + private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { + if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) + return; + + // We've waited ages for AT to be confirmed into a block but something has gone awry. + // After this long we assume transaction loss so give up with trade-bot entry too. + tradeBotData.setState(State.BOB_REFUNDED.name()); + tradeBotData.setStateValue(State.BOB_REFUNDED.value); + tradeBotData.setTimestamp(NTP.getTime()); + // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + repository.saveChanges(); + + LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); + TradeBot.notifyStateChange(tradeBotData); + return; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, + () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for Alice's P2SH-A to confirm. + *

+ * If P2SH-A is confirmed, then trade-bot's next step is to MESSAGE Bob's trade address with Alice's trade info. + *

+ * It is possible between broadcast and confirmation of P2SH-A funding transaction, that Bob has cancelled his trade offer. + * If this is detected then trade-bot's next step is to wait until P2SH-A can refund back to Alice. + *

+ * In normal operation, trade-bot send a zero-fee, PoW MESSAGE on Alice's behalf containing: + *

    + *
  • Alice's 'foreign'/Bitcoin public key hash - so Bob's trade-bot can derive P2SH-A address and check balance
    • + *
  • HASH160 of Alice's secret-A - also used to derive P2SH-A address
    • + *
  • lockTime of P2SH-A - also used to derive P2SH-A address, but also for other use later in the trading process
    • + *
+ * If MESSAGE transaction is successfully broadcast, trade-bot's next step is to wait until Bob's AT has locked trade to Alice only. + * @throws ForeignBlockchainException + */ + private void handleAliceWaitingForP2shA(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Bitcoin bitcoin = Bitcoin.getInstance(); + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getLockTimeA(), crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestampA = calcP2shAFeeTimestamp(tradeBotData.getLockTimeA(), crossChainTradeData.tradeTimeout); + long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); + long minimumAmountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // This shouldn't occur, but defensively check P2SH-B in case we haven't redeemed the AT + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WATCH_P2SH_B, + () -> String.format("P2SH-A %s already spent? Defensively checking P2SH-B next", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); + return; + + case FUNDED: + // Fall-through out of switch... + break; + } + + // P2SH-A funding confirmed + + // Attempt to send MESSAGE to Bob's Qortal trade address + byte[] messageData = BitcoinACCTv1.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); + String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WAITING_FOR_AT_LOCK, + () -> String.format("P2SH-A %s funding confirmed. Messaged %s. Waiting for AT %s to lock to us", + p2shAddressA, messageRecipient, tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. + *

+ * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, + * in which case trade-bot is done with this specific trade and finalizes on refunded state. + *

+ * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. + *

+ * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. + *

+ * Assuming P2SH-A has at least expected Bitcoin balance, + * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. + *

+ * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. + *

+ * Trade-bot's next step is to wait for P2SH-B, which will allow Bob to reveal his secret-B, + * needed by Alice to progress her side of the trade. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // If AT has finished then Bob likely cancelled his trade offer + if (atData.getIsFinished()) { + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); + return; + } + + Bitcoin bitcoin = Bitcoin.getInstance(); + + String address = tradeBotData.getTradeNativeAddress(); + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); + + final byte[] originalLastTransactionSignature = tradeBotData.getLastTransactionSignature(); + + // Skip past previously processed messages + if (originalLastTransactionSignature != null) + for (int i = 0; i < messageTransactionsData.size(); ++i) + if (Arrays.equals(messageTransactionsData.get(i).getSignature(), originalLastTransactionSignature)) { + messageTransactionsData.subList(0, i + 1).clear(); + break; + } + + while (!messageTransactionsData.isEmpty()) { + MessageTransactionData messageTransactionData = messageTransactionsData.remove(0); + tradeBotData.setLastTransactionSignature(messageTransactionData.getSignature()); + + if (messageTransactionData.isText()) + continue; + + // We're expecting: HASH160(secret-A), Alice's Bitcoin pubkeyhash and lockTime-A + byte[] messageData = messageTransactionData.getData(); + BitcoinACCTv1.OfferMessageData offerMessageData = BitcoinACCTv1.extractOfferMessageData(messageData); + if (offerMessageData == null) + continue; + + byte[] aliceForeignPublicKeyHash = offerMessageData.partnerBitcoinPKH; + byte[] hashOfSecretA = offerMessageData.hashOfSecretA; + int lockTimeA = (int) offerMessageData.lockTimeA; + + // Determine P2SH-A address and confirm funded + byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); + String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); + + long feeTimestampA = calcP2shAFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); + final long minimumAmountA = tradeBotData.getForeignAmount() - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // There might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // This shouldn't occur, but defensively bump to next state + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_P2SH_B, + () -> String.format("P2SH-A %s already spent? Defensively checking P2SH-B next", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case FUNDED: + // Fall-through out of switch... + break; + } + + // Good to go - send MESSAGE to AT + + String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(messageTransactionData.getTimestamp(), lockTimeA); + + // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume + byte[] outgoingMessageData = BitcoinACCTv1.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); + + outgoingMessageTransaction.computeNonce(); + outgoingMessageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + byte[] redeemScriptB = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeB, tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret()); + String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_P2SH_B, + () -> String.format("Locked AT %s to %s. Waiting for P2SH-B %s", tradeBotData.getAtAddress(), aliceNativeAddress, p2shAddressB)); + + return; + } + + // Don't resave/notify if we don't need to + if (tradeBotData.getLastTransactionSignature() != originalLastTransactionSignature) + TradeBot.updateTradeBotState(repository, tradeBotData, null); + } + + /** + * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. + *

+ * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow + * this process has taken so long that we've reached P2SH-A's locktime, or that someone else + * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. + *

+ * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. + *

+ * If all is well, trade-bot then uses Bitcoin wallet to (token) fund P2SH-B. + *

+ * If P2SH-B funding transaction is successfully broadcast to the Bitcoin network, trade-bot's next + * step is to watch for Bob revealing secret-B by redeeming P2SH-B. + * @throws ForeignBlockchainException + */ + private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Bitcoin bitcoin = Bitcoin.getInstance(); + int lockTimeA = tradeBotData.getLockTimeA(); + + // Refund P2SH-A if we've passed lockTime-A + if (NTP.getTime() >= tradeBotData.getLockTimeA() * 1000L) { + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); + + long feeTimestampA = calcP2shAFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); + long minimumAmountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // This shouldn't occur, but defensively revert back to waiting for P2SH-A + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WAITING_FOR_P2SH_A, + () -> String.format("P2SH-A %s no longer funded? Defensively checking P2SH-A next", p2shAddressA)); + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // This shouldn't occur, but defensively bump to next state + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WATCH_P2SH_B, + () -> String.format("P2SH-A %s already spent? Defensively checking P2SH-B next", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); + return; + + case FUNDED: + // Fall-through out of switch... + break; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> atData.getIsFinished() + ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) + : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); + + return; + } + + // We're waiting for AT to be in TRADE mode + if (crossChainTradeData.mode != AcctMode.TRADING) + return; + + // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above + + // Alice needs to fund P2SH-B here + + // Find our MESSAGE to AT from previous state + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), + crossChainTradeData.qortalCreatorTradeAddress, null, null, null); + if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { + LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); + return; + } + + long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(recipientMessageTimestamp, lockTimeA); + + // Our calculated lockTime-B should match AT's calculated lockTime-B + if (lockTimeB != crossChainTradeData.lockTimeB) { + LOGGER.debug(() -> String.format("Trade AT lockTime-B '%d' doesn't match our lockTime-B '%d'", crossChainTradeData.lockTimeB, lockTimeB)); + // We'll eventually refund + return; + } + + byte[] redeemScriptB = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeB, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretB); + String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); + + long feeTimestampB = calcP2shBFeeTimestamp(lockTimeA, lockTimeB); + long p2shFeeB = bitcoin.getP2shFee(feeTimestampB); + + // Have we funded P2SH-B already? + final long minimumAmountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB; + + BitcoinyHTLC.Status htlcStatusB = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressB, minimumAmountB); + + switch (htlcStatusB) { + case UNFUNDED: { + // Do not include fee for funding transaction as this is covered by buildSpend() + long amountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB /*redeeming/refunding P2SH-B*/; + + Transaction p2shFundingTransaction = bitcoin.buildSpend(tradeBotData.getForeignKey(), p2shAddressB, amountB); + if (p2shFundingTransaction == null) { + LOGGER.debug("Unable to build P2SH-B funding transaction - lack of funds?"); + return; + } + + bitcoin.broadcastTransaction(p2shFundingTransaction); + break; + } + + case FUNDING_IN_PROGRESS: + case FUNDED: + break; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // This shouldn't occur, but defensively bump to next state + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WATCH_P2SH_B, + () -> String.format("P2SH-B %s already spent? Defensively checking P2SH-B next", p2shAddressB)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("P2SH-B %s already refunded. Refunding P2SH-A next", p2shAddressB)); + return; + } + + // P2SH-B funded, now we wait for Bob to redeem it + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WATCH_P2SH_B, + () -> String.format("AT %s locked to us (%s). P2SH-B %s funded. Watching P2SH-B for secret-B", + tradeBotData.getAtAddress(), tradeBotData.getTradeNativeAddress(), p2shAddressB)); + } + + /** + * Trade-bot is waiting for P2SH-B to funded. + *

+ * It's possible than Bob's AT has reached it's trading timeout and automatically refunded QORT back to Bob. + * In which case, trade-bot is done with this specific trade and finalizes on refunded state. + *

+ * Assuming P2SH-B is funded, trade-bot 'redeems' this P2SH using secret-B, thus revealing it to Alice. + *

+ * Trade-bot's next step is to wait for Alice to use secret-B, and her secret-A, to redeem Bob's AT. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForP2shB(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // If we've passed AT refund timestamp then AT will have finished after auto-refunding + if (atData.getIsFinished()) { + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); + + return; + } + + // It's possible AT hasn't processed our previous MESSAGE yet and so lockTimeB won't be set + if (crossChainTradeData.lockTimeB == null) + // AT yet to process MESSAGE + return; + + Bitcoin bitcoin = Bitcoin.getInstance(); + + byte[] redeemScriptB = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, crossChainTradeData.lockTimeB, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretB); + String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); + + long feeTimestampB = calcP2shBFeeTimestamp(crossChainTradeData.lockTimeA, crossChainTradeData.lockTimeB); + long p2shFeeB = bitcoin.getP2shFee(feeTimestampB); + + final long minimumAmountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB; + + BitcoinyHTLC.Status htlcStatusB = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressB, minimumAmountB); + + switch (htlcStatusB) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-B to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // This shouldn't occur, but defensively bump to next state + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, + () -> String.format("P2SH-B %s already spent (exposing secret-B)? Checking AT %s for secret-A", p2shAddressB, tradeBotData.getAtAddress())); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // AT should auto-refund - we don't need to do anything here + return; + + case FUNDED: + break; + } + + // Redeem P2SH-B using secret-B + Coin redeemAmount = Coin.valueOf(P2SH_B_OUTPUT_AMOUNT); // An actual amount to avoid dust filter, remaining used as fees. The real funds are in P2SH-A. + ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = bitcoin.getUnspentOutputs(p2shAddressB); + byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(bitcoin.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptB, tradeBotData.getSecret(), receivingAccountInfo); + + bitcoin.broadcastTransaction(p2shRedeemTransaction); + + // P2SH-B redeemed, now we wait for Alice to use secret-A to redeem AT + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, + () -> String.format("P2SH-B %s redeemed (exposing secret-B). Watching AT %s for secret-A", p2shAddressB, tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for Bob to redeem P2SH-B thus revealing secret-B to Alice. + *

+ * It's possible that this process has taken so long that we've reached P2SH-B's locktime. + * In which case, trade-bot switches to begin the refund process. + *

+ * If trade-bot can extract a valid secret-B from the spend of P2SH-B, then it creates a + * zero-fee, PoW MESSAGE to send to Bob's AT, including both secret-B and also Alice's secret-A. + *

+ * Both secrets are needed to release the QORT funds from Bob's AT to Alice's 'native'/Qortal + * trade address. + *

+ * In revealing a valid secret-A, Bob can then redeem the BTC funds from P2SH-A. + *

+ * If trade-bot successfully broadcasts the MESSAGE transaction, then this specific trade is done. + * @throws ForeignBlockchainException + */ + private void handleAliceWatchingP2shB(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Bitcoin bitcoin = Bitcoin.getInstance(); + + byte[] redeemScriptB = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), crossChainTradeData.lockTimeB, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretB); + String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); + + long feeTimestampB = calcP2shBFeeTimestamp(crossChainTradeData.lockTimeA, crossChainTradeData.lockTimeB); + long p2shFeeB = bitcoin.getP2shFee(feeTimestampB); + final long minimumAmountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB; + + BitcoinyHTLC.Status htlcStatusB = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressB, minimumAmountB); + + switch (htlcStatusB) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + case FUNDED: + case REDEEM_IN_PROGRESS: + // Still waiting for P2SH-B to be funded/redeemed... + return; + + case REDEEMED: + // Bob has redeemed P2SH-B, so double-check that we have redeemed AT... + break; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // We've refunded P2SH-B? Bump to refunding P2SH-A then + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("P2SH-B %s already refunded. Refunding P2SH-A next", p2shAddressB)); + return; + } + + byte[] secretB = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddressB); + if (secretB == null) + // Secret not revealed at this time + return; + + // Send 'redeem' MESSAGE to AT using both secrets + byte[] secretA = tradeBotData.getSecret(); + String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH + byte[] messageData = BitcoinACCTv1.buildRedeemMessage(secretA, secretB, qortalReceivingAddress); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // Reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-B %s redeemed, using secrets to redeem AT %s. Funds should arrive at %s", + p2shAddressB, tradeBotData.getAtAddress(), qortalReceivingAddress)); + } + + /** + * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the BTC funds from P2SH-A. + *

+ * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, + * trade-bot is done with this specific trade and finalizes in refunded state. + *

+ * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the BTC funds from P2SH-A + * to Bob's 'foreign'/Bitcoin trade legacy-format address, as derived from trade private key. + *

+ * (This could potentially be 'improved' to send BTC to any address of Bob's choosing by changing the transaction output). + *

+ * If trade-bot successfully broadcasts the transaction, then this specific trade is done. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // AT should be 'finished' once Alice has redeemed QORT funds + if (!atData.getIsFinished()) + // Not finished yet + return; + + // If AT is not REDEEMED then something has gone wrong + if (crossChainTradeData.mode != AcctMode.REDEEMED) { + // Not redeemed so must be refunded/cancelled + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); + + return; + } + + byte[] secretA = BitcoinACCTv1.getInstance().findSecretA(repository, crossChainTradeData); + if (secretA == null) { + LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); + return; + } + + // Use secret-A to redeem P2SH-A + + Bitcoin bitcoin = Bitcoin.getInstance(); + int lockTimeA = crossChainTradeData.lockTimeA; + + byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); + String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestampA = calcP2shAFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); + long minimumAmountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Double-check that we have redeemed P2SH-A... + break; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // Wait for AT to auto-refund + return; + + case FUNDED: { + Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT); + ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = bitcoin.getUnspentOutputs(p2shAddressA); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(bitcoin.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); + + bitcoin.broadcastTransaction(p2shRedeemTransaction); + break; + } + } + + String receivingAddress = bitcoin.pkhToAddress(receivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); + } + + /** + * Trade-bot is attempting to refund P2SH-B. + *

+ * We could potentially skip this step as P2SH-B is only funded with a token amount to cover the mining fee should Bob redeem P2SH-B. + *

+ * Upon successful broadcast of P2SH-B refunding transaction, trade-bot's next step is to begin refunding of P2SH-A. + * @throws ForeignBlockchainException + */ + private void handleAliceRefundingP2shB(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + int lockTimeB = crossChainTradeData.lockTimeB; + + // We can't refund P2SH-B until lockTime-B has passed + if (NTP.getTime() <= lockTimeB * 1000L) + return; + + Bitcoin bitcoin = Bitcoin.getInstance(); + + // We can't refund P2SH-B until median block time has passed lockTime-B (see BIP113) + int medianBlockTime = bitcoin.getMedianBlockTime(); + if (medianBlockTime <= lockTimeB) + return; + + byte[] redeemScriptB = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeB, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretB); + String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); + + long feeTimestampB = calcP2shBFeeTimestamp(crossChainTradeData.lockTimeA, lockTimeB); + long p2shFeeB = bitcoin.getP2shFee(feeTimestampB); + final long minimumAmountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB; + + BitcoinyHTLC.Status htlcStatusB = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressB, minimumAmountB); + + switch (htlcStatusB) { + case UNFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("P2SH-B %s never funded?. Refunding P2SH-A next", p2shAddressB)); + return; + + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-B to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // We must be very close to trade timeout. Defensively try to refund P2SH-A + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("P2SH-B %s already spent?. Refunding P2SH-A next", p2shAddressB)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + break; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(P2SH_B_OUTPUT_AMOUNT); // An actual amount to avoid dust filter, remaining used as fees. + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = bitcoin.getUnspentOutputs(p2shAddressB); + + // Determine receive address for refund + String receiveAddress = bitcoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + Address receiving = Address.fromString(bitcoin.getNetworkParameters(), receiveAddress); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoin.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptB, lockTimeB, receiving.getHash()); + + bitcoin.broadcastTransaction(p2shRefundTransaction); + break; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("Refunded P2SH-B %s. Waiting for LockTime-A", p2shAddressB)); + } + + /** + * Trade-bot is attempting to refund P2SH-A. + * @throws ForeignBlockchainException + */ + private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + int lockTimeA = tradeBotData.getLockTimeA(); + + // We can't refund P2SH-A until lockTime-A has passed + if (NTP.getTime() <= lockTimeA * 1000L) + return; + + Bitcoin bitcoin = Bitcoin.getInstance(); + + // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) + int medianBlockTime = bitcoin.getMedianBlockTime(); + if (medianBlockTime <= lockTimeA) + return; + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestampA = calcP2shAFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); + long minimumAmountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-A to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Too late! + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent!", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + break; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT); + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = bitcoin.getUnspentOutputs(p2shAddressA); + + // Determine receive address for refund + String receiveAddress = bitcoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + Address receiving = Address.fromString(bitcoin.getNetworkParameters(), receiveAddress); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoin.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); + + bitcoin.broadcastTransaction(p2shRefundTransaction); + break; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); + } + + /** + * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. + *

+ * Will automatically update trade-bot state to ALICE_REFUNDING_B or ALICE_DONE as necessary. + * + * @throws DataException + * @throws ForeignBlockchainException + */ + private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // This is OK + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) + return false; + + boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); + + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING && isAtLockedToUs) + return false; + + if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { + // We've redeemed already? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); + } else { + // Any other state is not good, so start defensive refund + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_B, + () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); + } + + return true; + } + + private long calcP2shAFeeTimestamp(int lockTimeA, int tradeTimeout) { + return (lockTimeA - tradeTimeout * 60) * 1000L; + } + + private long calcP2shBFeeTimestamp(int lockTimeA, int lockTimeB) { + // lockTimeB is halfway between offerMessageTimestamp and lockTimeA + return (lockTimeA - (lockTimeA - lockTimeB) * 2) * 1000L; + } + +} diff --git a/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv1TradeBot.java new file mode 100644 index 000000000..d37a66503 --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv1TradeBot.java @@ -0,0 +1,885 @@ +package org.qortal.controller.tradebot; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.*; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.asset.Asset; +import org.qortal.crosschain.*; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.DeployAtTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +/** + * Performing cross-chain trading steps on behalf of user. + *

+ * We deal with three different independent state-spaces here: + *

    + *
  • Qortal blockchain
    • + *
  • Foreign blockchain
    • + *
  • Trade-bot entries
    • + *
+ */ +public class DogecoinACCTv1TradeBot implements AcctTradeBot { + + private static final Logger LOGGER = LogManager.getLogger(DogecoinACCTv1TradeBot.class); + + public enum State implements TradeBot.StateNameAndValueSupplier { + BOB_WAITING_FOR_AT_CONFIRM(10, false, false), + BOB_WAITING_FOR_MESSAGE(15, true, true), + BOB_WAITING_FOR_AT_REDEEM(25, true, true), + BOB_DONE(30, false, false), + BOB_REFUNDED(35, false, false), + + ALICE_WAITING_FOR_AT_LOCK(85, true, true), + ALICE_DONE(95, false, false), + ALICE_REFUNDING_A(105, true, true), + ALICE_REFUNDED(110, false, false); + + private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); + + public final int value; + public final boolean requiresAtData; + public final boolean requiresTradeData; + + State(int value, boolean requiresAtData, boolean requiresTradeData) { + this.value = value; + this.requiresAtData = requiresAtData; + this.requiresTradeData = requiresTradeData; + } + + public static State valueOf(int value) { + return map.get(value); + } + + @Override + public String getState() { + return this.name(); + } + + @Override + public int getStateValue() { + return this.value; + } + } + + /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ + private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms + + private static DogecoinACCTv1TradeBot instance; + + private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream() + .map(State::name) + .collect(Collectors.toUnmodifiableList()); + + private DogecoinACCTv1TradeBot() { + } + + public static synchronized DogecoinACCTv1TradeBot getInstance() { + if (instance == null) + instance = new DogecoinACCTv1TradeBot(); + + return instance; + } + + @Override + public List getEndStates() { + return this.endStates; + } + + /** + * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for DOGE. + *

+ * Generates: + *

    + *
  • new 'trade' private key
    • + *
+ * Derives: + *
    + *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • + *
  • 'foreign' (as in Dogecoin) public key, public key hash
    • + *
+ * A Qortal AT is then constructed including the following as constants in the 'data segment': + *
    + *
  • 'native'/Qortal 'trade' address - used as a MESSAGE contact
    • + *
  • 'foreign'/Dogecoin public key hash - used by Alice's P2SH scripts to allow redeem
    • + *
  • QORT amount on offer by Bob
    • + *
  • DOGE amount expected in return by Bob (from Alice)
    • + *
  • trading timeout, in case things go wrong and everyone needs to refund
    • + *
+ * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. + *

+ * Trade-bot will wait for Bob's AT to be deployed before taking next step. + *

+ * @param repository + * @param tradeBotCreateRequest + * @return raw, unsigned DEPLOY_AT transaction + * @throws DataException + */ + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + + // Convert Dogecoin receiving address into public key hash (we only support P2PKH at this time) + Address dogecoinReceivingAddress; + try { + dogecoinReceivingAddress = Address.fromString(Dogecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); + } catch (AddressFormatException e) { + throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + } + if (dogecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) + throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + + byte[] dogecoinReceivingAccountInfo = dogecoinReceivingAddress.getHash(); + + PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); + + // Deploy AT + long timestamp = NTP.getTime(); + byte[] reference = creator.getLastReference(); + long fee = 0L; + byte[] signature = null; + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); + + String name = "QORT/DOGE ACCT"; + String description = "QORT/DOGE cross-chain trade"; + String aTType = "ACCT"; + String tags = "ACCT QORT DOGE"; + byte[] creationBytes = DogecoinACCTv1.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount, + tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); + long amount = tradeBotCreateRequest.fundingQortAmount; + + DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + DeployAtTransaction.ensureATAddress(deployAtTransactionData); + String atAddress = deployAtTransactionData.getAtAddress(); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, DogecoinACCTv1.NAME, + State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, + creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + null, null, + SupportedBlockchain.DOGECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + tradeBotCreateRequest.foreignAmount, null, null, null, dogecoinReceivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); + + // Attempt to backup the trade bot data + TradeBot.backupTradeBotData(repository, null); + + // Return to user for signing and broadcast as we don't have their Qortal private key + try { + return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); + } catch (TransformationException e) { + throw new DataException("Failed to transform DEPLOY_AT transaction?", e); + } + } + + /** + * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching DOGE to an existing offer. + *

+ * Requires a chosen trade offer from Bob, passed by crossChainTradeData + * and access to a Dogecoin wallet via xprv58. + *

+ * The crossChainTradeData contains the current trade offer state + * as extracted from the AT's data segment. + *

+ * Access to a funded wallet is via a Dogecoin BIP32 hierarchical deterministic key, + * passed via xprv58. + * This key will be stored in your node's database + * to allow trade-bot to create/fund the necessary P2SH transactions! + * However, due to the nature of BIP32 keys, it is possible to give the trade-bot + * only a subset of wallet access (see BIP32 for more details). + *

+ * As an example, the xprv58 can be extract from a legacy, password-less + * Electrum wallet by going to the console tab and entering:
+ * wallet.keystore.xprv
+ * which should result in a base58 string starting with either 'xprv' (for Dogecoin main-net) + * or 'tprv' for (Dogecoin test-net). + *

+ * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. + *

+ * If sufficient funds are available, this method will actually fund the P2SH-A + * with the Dogecoin amount expected by 'Bob'. + *

+ * If the Dogecoin transaction is successfully broadcast to the network then + * we also send a MESSAGE to Bob's trade-bot to let them know. + *

+ * The trade-bot entry is saved to the repository and the cross-chain trading process commences. + *

+ * @param repository + * @param crossChainTradeData chosen trade OFFER that Alice wants to match + * @param xprv58 funded wallet xprv in base58 + * @return true if P2SH-A funding transaction successfully broadcast to Dogecoin network, false otherwise + * @throws DataException + */ + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + byte[] secretA = TradeBot.generateSecret(); + byte[] hashOfSecretA = Crypto.hash160(secretA); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH + + // We need to generate lockTime-A: add tradeTimeout to now + long now = NTP.getTime(); + int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, DogecoinACCTv1.NAME, + State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value, + receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secretA, hashOfSecretA, + SupportedBlockchain.DOGECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); + + // Attempt to backup the trade bot data + // Include tradeBotData as an additional parameter, since it's not in the repository yet + TradeBot.backupTradeBotData(repository, Arrays.asList(tradeBotData)); + + // Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount + long p2shFee; + try { + p2shFee = Dogecoin.getInstance().getP2shFee(now); + } catch (ForeignBlockchainException e) { + LOGGER.debug("Couldn't estimate Dogecoin fees?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Fee for redeem/refund is subtracted from P2SH-A balance. + // Do not include fee for funding transaction as this is covered by buildSpend() + long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/; + + // P2SH-A to be funded + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); + String p2shAddress = Dogecoin.getInstance().deriveP2shAddress(redeemScriptBytes); + + // Build transaction for funding P2SH-A + Transaction p2shFundingTransaction = Dogecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); + if (p2shFundingTransaction == null) { + LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); + return ResponseResult.BALANCE_ISSUE; + } + + try { + Dogecoin.getInstance().broadcastTransaction(p2shFundingTransaction); + } catch (ForeignBlockchainException e) { + // We couldn't fund P2SH-A at this time + LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Attempt to send MESSAGE to Bob's Qortal trade address + byte[] messageData = DogecoinACCTv1.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); + String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); + return ResponseResult.NETWORK_ISSUE; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress)); + + return ResponseResult.OK; + } + + @Override + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) + return true; + + // If the AT doesn't exist then we might as well let the user tidy up + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) + return true; + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + case ALICE_DONE: + case BOB_DONE: + case ALICE_REFUNDED: + case BOB_REFUNDED: + case ALICE_REFUNDING_A: + return true; + + default: + return false; + } + } + + @Override + public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) { + LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); + return; + } + + ATData atData = null; + CrossChainTradeData tradeData = null; + + if (tradeBotState.requiresAtData) { + // Attempt to fetch AT data + atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); + if (atData == null) { + LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + return; + } + + if (tradeBotState.requiresTradeData) { + tradeData = DogecoinACCTv1.getInstance().populateTradeData(repository, atData); + if (tradeData == null) { + LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); + return; + } + } + } + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + handleBobWaitingForAtConfirm(repository, tradeBotData); + break; + + case BOB_WAITING_FOR_MESSAGE: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_WAITING_FOR_AT_LOCK: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_AT_REDEEM: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_DONE: + case BOB_DONE: + break; + + case ALICE_REFUNDING_A: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_REFUNDED: + case BOB_REFUNDED: + break; + } + } + + /** + * Trade-bot is waiting for Bob's AT to deploy. + *

+ * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. + */ + private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { + if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) + return; + + // We've waited ages for AT to be confirmed into a block but something has gone awry. + // After this long we assume transaction loss so give up with trade-bot entry too. + tradeBotData.setState(State.BOB_REFUNDED.name()); + tradeBotData.setStateValue(State.BOB_REFUNDED.value); + tradeBotData.setTimestamp(NTP.getTime()); + // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + repository.saveChanges(); + + LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); + TradeBot.notifyStateChange(tradeBotData); + return; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, + () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. + *

+ * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, + * in which case trade-bot is done with this specific trade and finalizes on refunded state. + *

+ * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. + *

+ * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. + *

+ * Assuming P2SH-A has at least expected Dogecoin balance, + * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. + *

+ * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. + *

+ * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to + * extract secret-A needed to redeem Alice's P2SH. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // If AT has finished then Bob likely cancelled his trade offer + if (atData.getIsFinished()) { + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); + return; + } + + Dogecoin dogecoin = Dogecoin.getInstance(); + + String address = tradeBotData.getTradeNativeAddress(); + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); + + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + if (messageTransactionData.isText()) + continue; + + // We're expecting: HASH160(secret-A), Alice's Dogecoin pubkeyhash and lockTime-A + byte[] messageData = messageTransactionData.getData(); + DogecoinACCTv1.OfferMessageData offerMessageData = DogecoinACCTv1.extractOfferMessageData(messageData); + if (offerMessageData == null) + continue; + + byte[] aliceForeignPublicKeyHash = offerMessageData.partnerDogecoinPKH; + byte[] hashOfSecretA = offerMessageData.hashOfSecretA; + int lockTimeA = (int) offerMessageData.lockTimeA; + long messageTimestamp = messageTransactionData.getTimestamp(); + int refundTimeout = DogecoinACCTv1.calcRefundTimeout(messageTimestamp, lockTimeA); + + // Determine P2SH-A address and confirm funded + byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); + String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); + final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // There might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // We've already redeemed this? + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case FUNDED: + // Fall-through out of switch... + break; + } + + // Good to go - send MESSAGE to AT + + String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); + + // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume + byte[] outgoingMessageData = DogecoinACCTv1.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); + + outgoingMessageTransaction.computeNonce(); + outgoingMessageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, + () -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress)); + + return; + } + } + + /** + * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. + *

+ * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow + * this process has taken so long that we've reached P2SH-A's locktime, or that someone else + * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. + *

+ * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. + *

+ * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice. + *

+ * In revealing a valid secret-A, Bob can then redeem the DOGE funds from P2SH-A. + *

+ * @throws ForeignBlockchainException + */ + private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Dogecoin dogecoin = Dogecoin.getInstance(); + int lockTimeA = tradeBotData.getLockTimeA(); + + // Refund P2SH-A if we've passed lockTime-A + if (NTP.getTime() >= lockTimeA * 1000L) { + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + case FUNDED: + break; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Already redeemed? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); + return; + + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> atData.getIsFinished() + ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) + : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); + + return; + } + + // We're waiting for AT to be in TRADE mode + if (crossChainTradeData.mode != AcctMode.TRADING) + return; + + // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above + + // Find our MESSAGE to AT from previous state + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), + crossChainTradeData.qortalCreatorTradeAddress, null, null, null); + if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { + LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); + return; + } + + long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); + int refundTimeout = DogecoinACCTv1.calcRefundTimeout(recipientMessageTimestamp, lockTimeA); + + // Our calculated refundTimeout should match AT's refundTimeout + if (refundTimeout != crossChainTradeData.refundTimeout) { + LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout)); + // We'll eventually refund + return; + } + + // We're good to redeem AT + + // Send 'redeem' MESSAGE to AT using both secret + byte[] secretA = tradeBotData.getSecret(); + String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH + byte[] messageData = DogecoinACCTv1.buildRedeemMessage(secretA, qortalReceivingAddress); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // Reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("Redeeming AT %s. Funds should arrive at %s", + tradeBotData.getAtAddress(), qortalReceivingAddress)); + } + + /** + * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the DOGE funds from P2SH-A. + *

+ * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, + * trade-bot is done with this specific trade and finalizes in refunded state. + *

+ * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the DOGE funds from P2SH-A + * to Bob's 'foreign'/Dogecoin trade legacy-format address, as derived from trade private key. + *

+ * (This could potentially be 'improved' to send DOGE to any address of Bob's choosing by changing the transaction output). + *

+ * If trade-bot successfully broadcasts the transaction, then this specific trade is done. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // AT should be 'finished' once Alice has redeemed QORT funds + if (!atData.getIsFinished()) + // Not finished yet + return; + + // If AT is REFUNDED or CANCELLED then something has gone wrong + if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) { + // Alice hasn't redeemed the QORT, so there is no point in trying to redeem the DOGE + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); + + return; + } + + byte[] secretA = DogecoinACCTv1.getInstance().findSecretA(repository, crossChainTradeData); + if (secretA == null) { + LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); + return; + } + + // Use secret-A to redeem P2SH-A + + Dogecoin dogecoin = Dogecoin.getInstance(); + + byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + int lockTimeA = crossChainTradeData.lockTimeA; + byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); + String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Double-check that we have redeemed P2SH-A... + break; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // Wait for AT to auto-refund + return; + + case FUNDED: { + Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(dogecoin.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); + + dogecoin.broadcastTransaction(p2shRedeemTransaction); + break; + } + } + + String receivingAddress = dogecoin.pkhToAddress(receivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); + } + + /** + * Trade-bot is attempting to refund P2SH-A. + * @throws ForeignBlockchainException + */ + private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + int lockTimeA = tradeBotData.getLockTimeA(); + + // We can't refund P2SH-A until lockTime-A has passed + if (NTP.getTime() <= lockTimeA * 1000L) + return; + + Dogecoin dogecoin = Dogecoin.getInstance(); + + // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) + int medianBlockTime = dogecoin.getMedianBlockTime(); + if (medianBlockTime <= lockTimeA) + return; + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-A to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Too late! + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent!", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + break; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA); + + // Determine receive address for refund + String receiveAddress = dogecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + Address receiving = Address.fromString(dogecoin.getNetworkParameters(), receiveAddress); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(dogecoin.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); + + dogecoin.broadcastTransaction(p2shRefundTransaction); + break; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); + } + + /** + * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. + *

+ * Will automatically update trade-bot state to ALICE_REFUNDING_A or ALICE_DONE as necessary. + * + * @throws DataException + * @throws ForeignBlockchainException + */ + private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // This is OK + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) + return false; + + boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); + + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING) + if (isAtLockedToUs) { + // AT is trading with us - OK + return false; + } else { + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress)); + + return true; + } + + if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { + // We've redeemed already? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); + } else { + // Any other state is not good, so start defensive refund + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); + } + + return true; + } + + private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { + return (lockTimeA - tradeTimeout * 60) * 1000L; + } + +} diff --git a/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv2TradeBot.java b/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv2TradeBot.java new file mode 100644 index 000000000..96dfd1b1d --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv2TradeBot.java @@ -0,0 +1,884 @@ +package org.qortal.controller.tradebot; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.*; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.asset.Asset; +import org.qortal.crosschain.*; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.DeployAtTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +/** + * Performing cross-chain trading steps on behalf of user. + *

+ * We deal with three different independent state-spaces here: + *

    + *
  • Qortal blockchain
    • + *
  • Foreign blockchain
    • + *
  • Trade-bot entries
    • + *
+ */ +public class DogecoinACCTv2TradeBot implements AcctTradeBot { + + private static final Logger LOGGER = LogManager.getLogger(DogecoinACCTv2TradeBot.class); + + public enum State implements TradeBot.StateNameAndValueSupplier { + BOB_WAITING_FOR_AT_CONFIRM(10, false, false), + BOB_WAITING_FOR_MESSAGE(15, true, true), + BOB_WAITING_FOR_AT_REDEEM(25, true, true), + BOB_DONE(30, false, false), + BOB_REFUNDED(35, false, false), + + ALICE_WAITING_FOR_AT_LOCK(85, true, true), + ALICE_DONE(95, false, false), + ALICE_REFUNDING_A(105, true, true), + ALICE_REFUNDED(110, false, false); + + private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); + + public final int value; + public final boolean requiresAtData; + public final boolean requiresTradeData; + + State(int value, boolean requiresAtData, boolean requiresTradeData) { + this.value = value; + this.requiresAtData = requiresAtData; + this.requiresTradeData = requiresTradeData; + } + + public static State valueOf(int value) { + return map.get(value); + } + + @Override + public String getState() { + return this.name(); + } + + @Override + public int getStateValue() { + return this.value; + } + } + + /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ + private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms + + private static DogecoinACCTv2TradeBot instance; + + private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream() + .map(State::name) + .collect(Collectors.toUnmodifiableList()); + + private DogecoinACCTv2TradeBot() { + } + + public static synchronized DogecoinACCTv2TradeBot getInstance() { + if (instance == null) + instance = new DogecoinACCTv2TradeBot(); + + return instance; + } + + @Override + public List getEndStates() { + return this.endStates; + } + + /** + * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for DOGE. + *

+ * Generates: + *

    + *
  • new 'trade' private key
    • + *
+ * Derives: + *
    + *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • + *
  • 'foreign' (as in Dogecoin) public key, public key hash
    • + *
+ * A Qortal AT is then constructed including the following as constants in the 'data segment': + *
    + *
  • 'native'/Qortal 'trade' address - used as a MESSAGE contact
    • + *
  • 'foreign'/Dogecoin public key hash - used by Alice's P2SH scripts to allow redeem
    • + *
  • QORT amount on offer by Bob
    • + *
  • DOGE amount expected in return by Bob (from Alice)
    • + *
  • trading timeout, in case things go wrong and everyone needs to refund
    • + *
+ * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. + *

+ * Trade-bot will wait for Bob's AT to be deployed before taking next step. + *

+ * @param repository + * @param tradeBotCreateRequest + * @return raw, unsigned DEPLOY_AT transaction + * @throws DataException + */ + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + + // Convert Dogecoin receiving address into public key hash (we only support P2PKH at this time) + Address dogecoinReceivingAddress; + try { + dogecoinReceivingAddress = Address.fromString(Dogecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); + } catch (AddressFormatException e) { + throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + } + if (dogecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) + throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + + byte[] dogecoinReceivingAccountInfo = dogecoinReceivingAddress.getHash(); + + PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); + + // Deploy AT + long timestamp = NTP.getTime(); + byte[] reference = creator.getLastReference(); + long fee = 0L; + byte[] signature = null; + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); + + String name = "QORT/DOGE ACCT"; + String description = "QORT/DOGE cross-chain trade"; + String aTType = "ACCT"; + String tags = "ACCT QORT DOGE"; + byte[] creationBytes = DogecoinACCTv2.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount, + tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); + long amount = tradeBotCreateRequest.fundingQortAmount; + + DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + DeployAtTransaction.ensureATAddress(deployAtTransactionData); + String atAddress = deployAtTransactionData.getAtAddress(); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, DogecoinACCTv2.NAME, + State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, + creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + null, null, + SupportedBlockchain.DOGECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + tradeBotCreateRequest.foreignAmount, null, null, null, dogecoinReceivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); + + // Attempt to backup the trade bot data + TradeBot.backupTradeBotData(repository, null); + + // Return to user for signing and broadcast as we don't have their Qortal private key + try { + return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); + } catch (TransformationException e) { + throw new DataException("Failed to transform DEPLOY_AT transaction?", e); + } + } + + /** + * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching DOGE to an existing offer. + *

+ * Requires a chosen trade offer from Bob, passed by crossChainTradeData + * and access to a Dogecoin wallet via xprv58. + *

+ * The crossChainTradeData contains the current trade offer state + * as extracted from the AT's data segment. + *

+ * Access to a funded wallet is via a Dogecoin BIP32 hierarchical deterministic key, + * passed via xprv58. + * This key will be stored in your node's database + * to allow trade-bot to create/fund the necessary P2SH transactions! + * However, due to the nature of BIP32 keys, it is possible to give the trade-bot + * only a subset of wallet access (see BIP32 for more details). + *

+ * As an example, the xprv58 can be extract from a legacy, password-less + * Electrum wallet by going to the console tab and entering:
+ * wallet.keystore.xprv
+ * which should result in a base58 string starting with either 'xprv' (for Dogecoin main-net) + * or 'tprv' for (Dogecoin test-net). + *

+ * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. + *

+ * If sufficient funds are available, this method will actually fund the P2SH-A + * with the Dogecoin amount expected by 'Bob'. + *

+ * If the Dogecoin transaction is successfully broadcast to the network then + * we also send a MESSAGE to Bob's trade-bot to let them know. + *

+ * The trade-bot entry is saved to the repository and the cross-chain trading process commences. + *

+ * @param repository + * @param crossChainTradeData chosen trade OFFER that Alice wants to match + * @param xprv58 funded wallet xprv in base58 + * @return true if P2SH-A funding transaction successfully broadcast to Dogecoin network, false otherwise + * @throws DataException + */ + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + byte[] secretA = TradeBot.generateSecret(); + byte[] hashOfSecretA = Crypto.hash160(secretA); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH + + // We need to generate lockTime-A: add tradeTimeout to now + long now = NTP.getTime(); + int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, DogecoinACCTv2.NAME, + State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value, + receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secretA, hashOfSecretA, + SupportedBlockchain.DOGECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); + + // Attempt to backup the trade bot data + // Include tradeBotData as an additional parameter, since it's not in the repository yet + TradeBot.backupTradeBotData(repository, Arrays.asList(tradeBotData)); + + // Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount + long p2shFee; + try { + p2shFee = Dogecoin.getInstance().getP2shFee(now); + } catch (ForeignBlockchainException e) { + LOGGER.debug("Couldn't estimate Dogecoin fees?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Fee for redeem/refund is subtracted from P2SH-A balance. + // Do not include fee for funding transaction as this is covered by buildSpend() + long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/; + + // P2SH-A to be funded + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); + String p2shAddress = Dogecoin.getInstance().deriveP2shAddress(redeemScriptBytes); + + // Build transaction for funding P2SH-A + Transaction p2shFundingTransaction = Dogecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); + if (p2shFundingTransaction == null) { + LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); + return ResponseResult.BALANCE_ISSUE; + } + + try { + Dogecoin.getInstance().broadcastTransaction(p2shFundingTransaction); + } catch (ForeignBlockchainException e) { + // We couldn't fund P2SH-A at this time + LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Attempt to send MESSAGE to Bob's Qortal trade address + byte[] messageData = DogecoinACCTv2.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); + String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); + return ResponseResult.NETWORK_ISSUE; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress)); + + return ResponseResult.OK; + } + + @Override + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) + return true; + + // If the AT doesn't exist then we might as well let the user tidy up + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) + return true; + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + case ALICE_DONE: + case BOB_DONE: + case ALICE_REFUNDED: + case BOB_REFUNDED: + return true; + + default: + return false; + } + } + + @Override + public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) { + LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); + return; + } + + ATData atData = null; + CrossChainTradeData tradeData = null; + + if (tradeBotState.requiresAtData) { + // Attempt to fetch AT data + atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); + if (atData == null) { + LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + return; + } + + if (tradeBotState.requiresTradeData) { + tradeData = DogecoinACCTv2.getInstance().populateTradeData(repository, atData); + if (tradeData == null) { + LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); + return; + } + } + } + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + handleBobWaitingForAtConfirm(repository, tradeBotData); + break; + + case BOB_WAITING_FOR_MESSAGE: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_WAITING_FOR_AT_LOCK: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_AT_REDEEM: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_DONE: + case BOB_DONE: + break; + + case ALICE_REFUNDING_A: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_REFUNDED: + case BOB_REFUNDED: + break; + } + } + + /** + * Trade-bot is waiting for Bob's AT to deploy. + *

+ * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. + */ + private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { + if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) + return; + + // We've waited ages for AT to be confirmed into a block but something has gone awry. + // After this long we assume transaction loss so give up with trade-bot entry too. + tradeBotData.setState(State.BOB_REFUNDED.name()); + tradeBotData.setStateValue(State.BOB_REFUNDED.value); + tradeBotData.setTimestamp(NTP.getTime()); + // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + repository.saveChanges(); + + LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); + TradeBot.notifyStateChange(tradeBotData); + return; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, + () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. + *

+ * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, + * in which case trade-bot is done with this specific trade and finalizes on refunded state. + *

+ * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. + *

+ * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. + *

+ * Assuming P2SH-A has at least expected Dogecoin balance, + * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. + *

+ * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. + *

+ * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to + * extract secret-A needed to redeem Alice's P2SH. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // If AT has finished then Bob likely cancelled his trade offer + if (atData.getIsFinished()) { + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); + return; + } + + Dogecoin dogecoin = Dogecoin.getInstance(); + + String address = tradeBotData.getTradeNativeAddress(); + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); + + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + if (messageTransactionData.isText()) + continue; + + // We're expecting: HASH160(secret-A), Alice's Dogecoin pubkeyhash and lockTime-A + byte[] messageData = messageTransactionData.getData(); + DogecoinACCTv2.OfferMessageData offerMessageData = DogecoinACCTv2.extractOfferMessageData(messageData); + if (offerMessageData == null) + continue; + + byte[] aliceForeignPublicKeyHash = offerMessageData.partnerDogecoinPKH; + byte[] hashOfSecretA = offerMessageData.hashOfSecretA; + int lockTimeA = (int) offerMessageData.lockTimeA; + long messageTimestamp = messageTransactionData.getTimestamp(); + int refundTimeout = DogecoinACCTv2.calcRefundTimeout(messageTimestamp, lockTimeA); + + // Determine P2SH-A address and confirm funded + byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); + String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); + final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // There might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // We've already redeemed this? + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case FUNDED: + // Fall-through out of switch... + break; + } + + // Good to go - send MESSAGE to AT + + String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); + + // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume + byte[] outgoingMessageData = DogecoinACCTv2.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); + + outgoingMessageTransaction.computeNonce(); + outgoingMessageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, + () -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress)); + + return; + } + } + + /** + * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. + *

+ * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow + * this process has taken so long that we've reached P2SH-A's locktime, or that someone else + * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. + *

+ * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. + *

+ * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice. + *

+ * In revealing a valid secret-A, Bob can then redeem the DOGE funds from P2SH-A. + *

+ * @throws ForeignBlockchainException + */ + private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Dogecoin dogecoin = Dogecoin.getInstance(); + int lockTimeA = tradeBotData.getLockTimeA(); + + // Refund P2SH-A if we've passed lockTime-A + if (NTP.getTime() >= lockTimeA * 1000L) { + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + case FUNDED: + break; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Already redeemed? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); + return; + + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> atData.getIsFinished() + ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) + : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); + + return; + } + + // We're waiting for AT to be in TRADE mode + if (crossChainTradeData.mode != AcctMode.TRADING) + return; + + // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above + + // Find our MESSAGE to AT from previous state + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), + crossChainTradeData.qortalCreatorTradeAddress, null, null, null); + if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { + LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); + return; + } + + long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); + int refundTimeout = DogecoinACCTv2.calcRefundTimeout(recipientMessageTimestamp, lockTimeA); + + // Our calculated refundTimeout should match AT's refundTimeout + if (refundTimeout != crossChainTradeData.refundTimeout) { + LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout)); + // We'll eventually refund + return; + } + + // We're good to redeem AT + + // Send 'redeem' MESSAGE to AT using both secret + byte[] secretA = tradeBotData.getSecret(); + String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH + byte[] messageData = DogecoinACCTv2.buildRedeemMessage(secretA, qortalReceivingAddress); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // Reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("Redeeming AT %s. Funds should arrive at %s", + tradeBotData.getAtAddress(), qortalReceivingAddress)); + } + + /** + * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the DOGE funds from P2SH-A. + *

+ * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, + * trade-bot is done with this specific trade and finalizes in refunded state. + *

+ * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the DOGE funds from P2SH-A + * to Bob's 'foreign'/Dogecoin trade legacy-format address, as derived from trade private key. + *

+ * (This could potentially be 'improved' to send DOGE to any address of Bob's choosing by changing the transaction output). + *

+ * If trade-bot successfully broadcasts the transaction, then this specific trade is done. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // AT should be 'finished' once Alice has redeemed QORT funds + if (!atData.getIsFinished()) + // Not finished yet + return; + + // If AT is REFUNDED or CANCELLED then something has gone wrong + if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) { + // Alice hasn't redeemed the QORT, so there is no point in trying to redeem the DOGE + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); + + return; + } + + byte[] secretA = DogecoinACCTv2.getInstance().findSecretA(repository, crossChainTradeData); + if (secretA == null) { + LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); + return; + } + + // Use secret-A to redeem P2SH-A + + Dogecoin dogecoin = Dogecoin.getInstance(); + + byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + int lockTimeA = crossChainTradeData.lockTimeA; + byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); + String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Double-check that we have redeemed P2SH-A... + break; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // Wait for AT to auto-refund + return; + + case FUNDED: { + Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(dogecoin.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); + + dogecoin.broadcastTransaction(p2shRedeemTransaction); + break; + } + } + + String receivingAddress = dogecoin.pkhToAddress(receivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); + } + + /** + * Trade-bot is attempting to refund P2SH-A. + * @throws ForeignBlockchainException + */ + private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + int lockTimeA = tradeBotData.getLockTimeA(); + + // We can't refund P2SH-A until lockTime-A has passed + if (NTP.getTime() <= lockTimeA * 1000L) + return; + + Dogecoin dogecoin = Dogecoin.getInstance(); + + // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) + int medianBlockTime = dogecoin.getMedianBlockTime(); + if (medianBlockTime <= lockTimeA) + return; + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-A to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Too late! + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent!", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + break; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA); + + // Determine receive address for refund + String receiveAddress = dogecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + Address receiving = Address.fromString(dogecoin.getNetworkParameters(), receiveAddress); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(dogecoin.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); + + dogecoin.broadcastTransaction(p2shRefundTransaction); + break; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); + } + + /** + * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. + *

+ * Will automatically update trade-bot state to ALICE_REFUNDING_A or ALICE_DONE as necessary. + * + * @throws DataException + * @throws ForeignBlockchainException + */ + private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // This is OK + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) + return false; + + boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); + + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING) + if (isAtLockedToUs) { + // AT is trading with us - OK + return false; + } else { + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress)); + + return true; + } + + if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { + // We've redeemed already? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); + } else { + // Any other state is not good, so start defensive refund + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); + } + + return true; + } + + private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { + return (lockTimeA - tradeTimeout * 60) * 1000L; + } + +} diff --git a/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv3TradeBot.java b/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv3TradeBot.java new file mode 100644 index 000000000..996097f3a --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv3TradeBot.java @@ -0,0 +1,885 @@ +package org.qortal.controller.tradebot; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.*; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.asset.Asset; +import org.qortal.crosschain.*; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.DeployAtTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +/** + * Performing cross-chain trading steps on behalf of user. + *

+ * We deal with three different independent state-spaces here: + *

    + *
  • Qortal blockchain
    • + *
  • Foreign blockchain
    • + *
  • Trade-bot entries
    • + *
+ */ +public class DogecoinACCTv3TradeBot implements AcctTradeBot { + + private static final Logger LOGGER = LogManager.getLogger(DogecoinACCTv3TradeBot.class); + + public enum State implements TradeBot.StateNameAndValueSupplier { + BOB_WAITING_FOR_AT_CONFIRM(10, false, false), + BOB_WAITING_FOR_MESSAGE(15, true, true), + BOB_WAITING_FOR_AT_REDEEM(25, true, true), + BOB_DONE(30, false, false), + BOB_REFUNDED(35, false, false), + + ALICE_WAITING_FOR_AT_LOCK(85, true, true), + ALICE_DONE(95, false, false), + ALICE_REFUNDING_A(105, true, true), + ALICE_REFUNDED(110, false, false); + + private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); + + public final int value; + public final boolean requiresAtData; + public final boolean requiresTradeData; + + State(int value, boolean requiresAtData, boolean requiresTradeData) { + this.value = value; + this.requiresAtData = requiresAtData; + this.requiresTradeData = requiresTradeData; + } + + public static State valueOf(int value) { + return map.get(value); + } + + @Override + public String getState() { + return this.name(); + } + + @Override + public int getStateValue() { + return this.value; + } + } + + /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ + private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms + + private static DogecoinACCTv3TradeBot instance; + + private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream() + .map(State::name) + .collect(Collectors.toUnmodifiableList()); + + private DogecoinACCTv3TradeBot() { + } + + public static synchronized DogecoinACCTv3TradeBot getInstance() { + if (instance == null) + instance = new DogecoinACCTv3TradeBot(); + + return instance; + } + + @Override + public List getEndStates() { + return this.endStates; + } + + /** + * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for DOGE. + *

+ * Generates: + *

    + *
  • new 'trade' private key
    • + *
+ * Derives: + *
    + *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • + *
  • 'foreign' (as in Dogecoin) public key, public key hash
    • + *
+ * A Qortal AT is then constructed including the following as constants in the 'data segment': + *
    + *
  • 'native'/Qortal 'trade' address - used as a MESSAGE contact
    • + *
  • 'foreign'/Dogecoin public key hash - used by Alice's P2SH scripts to allow redeem
    • + *
  • QORT amount on offer by Bob
    • + *
  • DOGE amount expected in return by Bob (from Alice)
    • + *
  • trading timeout, in case things go wrong and everyone needs to refund
    • + *
+ * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. + *

+ * Trade-bot will wait for Bob's AT to be deployed before taking next step. + *

+ * @param repository + * @param tradeBotCreateRequest + * @return raw, unsigned DEPLOY_AT transaction + * @throws DataException + */ + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + + // Convert Dogecoin receiving address into public key hash (we only support P2PKH at this time) + Address dogecoinReceivingAddress; + try { + dogecoinReceivingAddress = Address.fromString(Dogecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); + } catch (AddressFormatException e) { + throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + } + if (dogecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) + throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + + byte[] dogecoinReceivingAccountInfo = dogecoinReceivingAddress.getHash(); + + PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); + + // Deploy AT + long timestamp = NTP.getTime(); + byte[] reference = creator.getLastReference(); + long fee = 0L; + byte[] signature = null; + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); + + String name = "QORT/DOGE ACCT"; + String description = "QORT/DOGE cross-chain trade"; + String aTType = "ACCT"; + String tags = "ACCT QORT DOGE"; + byte[] creationBytes = DogecoinACCTv3.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount, + tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); + long amount = tradeBotCreateRequest.fundingQortAmount; + + DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + DeployAtTransaction.ensureATAddress(deployAtTransactionData); + String atAddress = deployAtTransactionData.getAtAddress(); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, DogecoinACCTv3.NAME, + State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, + creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + null, null, + SupportedBlockchain.DOGECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + tradeBotCreateRequest.foreignAmount, null, null, null, dogecoinReceivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); + + // Attempt to backup the trade bot data + TradeBot.backupTradeBotData(repository, null); + + // Return to user for signing and broadcast as we don't have their Qortal private key + try { + return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); + } catch (TransformationException e) { + throw new DataException("Failed to transform DEPLOY_AT transaction?", e); + } + } + + /** + * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching DOGE to an existing offer. + *

+ * Requires a chosen trade offer from Bob, passed by crossChainTradeData + * and access to a Dogecoin wallet via xprv58. + *

+ * The crossChainTradeData contains the current trade offer state + * as extracted from the AT's data segment. + *

+ * Access to a funded wallet is via a Dogecoin BIP32 hierarchical deterministic key, + * passed via xprv58. + * This key will be stored in your node's database + * to allow trade-bot to create/fund the necessary P2SH transactions! + * However, due to the nature of BIP32 keys, it is possible to give the trade-bot + * only a subset of wallet access (see BIP32 for more details). + *

+ * As an example, the xprv58 can be extract from a legacy, password-less + * Electrum wallet by going to the console tab and entering:
+ * wallet.keystore.xprv
+ * which should result in a base58 string starting with either 'xprv' (for Dogecoin main-net) + * or 'tprv' for (Dogecoin test-net). + *

+ * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. + *

+ * If sufficient funds are available, this method will actually fund the P2SH-A + * with the Dogecoin amount expected by 'Bob'. + *

+ * If the Dogecoin transaction is successfully broadcast to the network then + * we also send a MESSAGE to Bob's trade-bot to let them know. + *

+ * The trade-bot entry is saved to the repository and the cross-chain trading process commences. + *

+ * @param repository + * @param crossChainTradeData chosen trade OFFER that Alice wants to match + * @param xprv58 funded wallet xprv in base58 + * @return true if P2SH-A funding transaction successfully broadcast to Dogecoin network, false otherwise + * @throws DataException + */ + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + byte[] secretA = TradeBot.generateSecret(); + byte[] hashOfSecretA = Crypto.hash160(secretA); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH + + // We need to generate lockTime-A: add tradeTimeout to now + long now = NTP.getTime(); + int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, DogecoinACCTv3.NAME, + State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value, + receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secretA, hashOfSecretA, + SupportedBlockchain.DOGECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); + + // Attempt to backup the trade bot data + // Include tradeBotData as an additional parameter, since it's not in the repository yet + TradeBot.backupTradeBotData(repository, Arrays.asList(tradeBotData)); + + // Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount + long p2shFee; + try { + p2shFee = Dogecoin.getInstance().getP2shFee(now); + } catch (ForeignBlockchainException e) { + LOGGER.debug("Couldn't estimate Dogecoin fees?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Fee for redeem/refund is subtracted from P2SH-A balance. + // Do not include fee for funding transaction as this is covered by buildSpend() + long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/; + + // P2SH-A to be funded + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); + String p2shAddress = Dogecoin.getInstance().deriveP2shAddress(redeemScriptBytes); + + // Build transaction for funding P2SH-A + Transaction p2shFundingTransaction = Dogecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); + if (p2shFundingTransaction == null) { + LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); + return ResponseResult.BALANCE_ISSUE; + } + + try { + Dogecoin.getInstance().broadcastTransaction(p2shFundingTransaction); + } catch (ForeignBlockchainException e) { + // We couldn't fund P2SH-A at this time + LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Attempt to send MESSAGE to Bob's Qortal trade address + byte[] messageData = DogecoinACCTv3.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); + String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); + return ResponseResult.NETWORK_ISSUE; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress)); + + return ResponseResult.OK; + } + + @Override + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) + return true; + + // If the AT doesn't exist then we might as well let the user tidy up + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) + return true; + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + case ALICE_DONE: + case BOB_DONE: + case ALICE_REFUNDED: + case BOB_REFUNDED: + case ALICE_REFUNDING_A: + return true; + + default: + return false; + } + } + + @Override + public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) { + LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); + return; + } + + ATData atData = null; + CrossChainTradeData tradeData = null; + + if (tradeBotState.requiresAtData) { + // Attempt to fetch AT data + atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); + if (atData == null) { + LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + return; + } + + if (tradeBotState.requiresTradeData) { + tradeData = DogecoinACCTv3.getInstance().populateTradeData(repository, atData); + if (tradeData == null) { + LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); + return; + } + } + } + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + handleBobWaitingForAtConfirm(repository, tradeBotData); + break; + + case BOB_WAITING_FOR_MESSAGE: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_WAITING_FOR_AT_LOCK: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_AT_REDEEM: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_DONE: + case BOB_DONE: + break; + + case ALICE_REFUNDING_A: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_REFUNDED: + case BOB_REFUNDED: + break; + } + } + + /** + * Trade-bot is waiting for Bob's AT to deploy. + *

+ * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. + */ + private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { + if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) + return; + + // We've waited ages for AT to be confirmed into a block but something has gone awry. + // After this long we assume transaction loss so give up with trade-bot entry too. + tradeBotData.setState(State.BOB_REFUNDED.name()); + tradeBotData.setStateValue(State.BOB_REFUNDED.value); + tradeBotData.setTimestamp(NTP.getTime()); + // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + repository.saveChanges(); + + LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); + TradeBot.notifyStateChange(tradeBotData); + return; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, + () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. + *

+ * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, + * in which case trade-bot is done with this specific trade and finalizes on refunded state. + *

+ * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. + *

+ * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. + *

+ * Assuming P2SH-A has at least expected Dogecoin balance, + * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. + *

+ * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. + *

+ * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to + * extract secret-A needed to redeem Alice's P2SH. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // If AT has finished then Bob likely cancelled his trade offer + if (atData.getIsFinished()) { + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); + return; + } + + Dogecoin dogecoin = Dogecoin.getInstance(); + + String address = tradeBotData.getTradeNativeAddress(); + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); + + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + if (messageTransactionData.isText()) + continue; + + // We're expecting: HASH160(secret-A), Alice's Dogecoin pubkeyhash and lockTime-A + byte[] messageData = messageTransactionData.getData(); + DogecoinACCTv3.OfferMessageData offerMessageData = DogecoinACCTv3.extractOfferMessageData(messageData); + if (offerMessageData == null) + continue; + + byte[] aliceForeignPublicKeyHash = offerMessageData.partnerDogecoinPKH; + byte[] hashOfSecretA = offerMessageData.hashOfSecretA; + int lockTimeA = (int) offerMessageData.lockTimeA; + long messageTimestamp = messageTransactionData.getTimestamp(); + int refundTimeout = DogecoinACCTv3.calcRefundTimeout(messageTimestamp, lockTimeA); + + // Determine P2SH-A address and confirm funded + byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); + String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); + final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // There might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // We've already redeemed this? + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case FUNDED: + // Fall-through out of switch... + break; + } + + // Good to go - send MESSAGE to AT + + String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); + + // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume + byte[] outgoingMessageData = DogecoinACCTv3.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); + + outgoingMessageTransaction.computeNonce(); + outgoingMessageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, + () -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress)); + + return; + } + } + + /** + * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. + *

+ * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow + * this process has taken so long that we've reached P2SH-A's locktime, or that someone else + * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. + *

+ * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. + *

+ * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice. + *

+ * In revealing a valid secret-A, Bob can then redeem the DOGE funds from P2SH-A. + *

+ * @throws ForeignBlockchainException + */ + private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Dogecoin dogecoin = Dogecoin.getInstance(); + int lockTimeA = tradeBotData.getLockTimeA(); + + // Refund P2SH-A if we've passed lockTime-A + if (NTP.getTime() >= lockTimeA * 1000L) { + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + case FUNDED: + break; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Already redeemed? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); + return; + + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> atData.getIsFinished() + ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) + : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); + + return; + } + + // We're waiting for AT to be in TRADE mode + if (crossChainTradeData.mode != AcctMode.TRADING) + return; + + // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above + + // Find our MESSAGE to AT from previous state + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), + crossChainTradeData.qortalCreatorTradeAddress, null, null, null); + if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { + LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); + return; + } + + long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); + int refundTimeout = DogecoinACCTv3.calcRefundTimeout(recipientMessageTimestamp, lockTimeA); + + // Our calculated refundTimeout should match AT's refundTimeout + if (refundTimeout != crossChainTradeData.refundTimeout) { + LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout)); + // We'll eventually refund + return; + } + + // We're good to redeem AT + + // Send 'redeem' MESSAGE to AT using both secret + byte[] secretA = tradeBotData.getSecret(); + String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH + byte[] messageData = DogecoinACCTv3.buildRedeemMessage(secretA, qortalReceivingAddress); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // Reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("Redeeming AT %s. Funds should arrive at %s", + tradeBotData.getAtAddress(), qortalReceivingAddress)); + } + + /** + * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the DOGE funds from P2SH-A. + *

+ * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, + * trade-bot is done with this specific trade and finalizes in refunded state. + *

+ * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the DOGE funds from P2SH-A + * to Bob's 'foreign'/Dogecoin trade legacy-format address, as derived from trade private key. + *

+ * (This could potentially be 'improved' to send DOGE to any address of Bob's choosing by changing the transaction output). + *

+ * If trade-bot successfully broadcasts the transaction, then this specific trade is done. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // AT should be 'finished' once Alice has redeemed QORT funds + if (!atData.getIsFinished()) + // Not finished yet + return; + + // If AT is REFUNDED or CANCELLED then something has gone wrong + if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) { + // Alice hasn't redeemed the QORT, so there is no point in trying to redeem the DOGE + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); + + return; + } + + byte[] secretA = DogecoinACCTv3.getInstance().findSecretA(repository, crossChainTradeData); + if (secretA == null) { + LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); + return; + } + + // Use secret-A to redeem P2SH-A + + Dogecoin dogecoin = Dogecoin.getInstance(); + + byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + int lockTimeA = crossChainTradeData.lockTimeA; + byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); + String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Double-check that we have redeemed P2SH-A... + break; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // Wait for AT to auto-refund + return; + + case FUNDED: { + Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(dogecoin.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); + + dogecoin.broadcastTransaction(p2shRedeemTransaction); + break; + } + } + + String receivingAddress = dogecoin.pkhToAddress(receivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); + } + + /** + * Trade-bot is attempting to refund P2SH-A. + * @throws ForeignBlockchainException + */ + private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + int lockTimeA = tradeBotData.getLockTimeA(); + + // We can't refund P2SH-A until lockTime-A has passed + if (NTP.getTime() <= lockTimeA * 1000L) + return; + + Dogecoin dogecoin = Dogecoin.getInstance(); + + // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) + int medianBlockTime = dogecoin.getMedianBlockTime(); + if (medianBlockTime <= lockTimeA) + return; + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-A to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Too late! + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent!", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + break; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA); + + // Determine receive address for refund + String receiveAddress = dogecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + Address receiving = Address.fromString(dogecoin.getNetworkParameters(), receiveAddress); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(dogecoin.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); + + dogecoin.broadcastTransaction(p2shRefundTransaction); + break; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); + } + + /** + * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. + *

+ * Will automatically update trade-bot state to ALICE_REFUNDING_A or ALICE_DONE as necessary. + * + * @throws DataException + * @throws ForeignBlockchainException + */ + private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // This is OK + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) + return false; + + boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); + + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING) + if (isAtLockedToUs) { + // AT is trading with us - OK + return false; + } else { + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress)); + + return true; + } + + if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { + // We've redeemed already? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); + } else { + // Any other state is not good, so start defensive refund + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); + } + + return true; + } + + private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { + return (lockTimeA - tradeTimeout * 60) * 1000L; + } + +} diff --git a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java new file mode 100644 index 000000000..fd0682b6c --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java @@ -0,0 +1,896 @@ +package org.qortal.controller.tradebot; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.Address; +import org.bitcoinj.core.AddressFormatException; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.ECKey; +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.TransactionOutput; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.asset.Asset; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.AcctMode; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.DeployAtTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +/** + * Performing cross-chain trading steps on behalf of user. + *

+ * We deal with three different independent state-spaces here: + *

    + *
  • Qortal blockchain
    • + *
  • Foreign blockchain
    • + *
  • Trade-bot entries
    • + *
+ */ +public class LitecoinACCTv1TradeBot implements AcctTradeBot { + + private static final Logger LOGGER = LogManager.getLogger(LitecoinACCTv1TradeBot.class); + + public enum State implements TradeBot.StateNameAndValueSupplier { + BOB_WAITING_FOR_AT_CONFIRM(10, false, false), + BOB_WAITING_FOR_MESSAGE(15, true, true), + BOB_WAITING_FOR_AT_REDEEM(25, true, true), + BOB_DONE(30, false, false), + BOB_REFUNDED(35, false, false), + + ALICE_WAITING_FOR_AT_LOCK(85, true, true), + ALICE_DONE(95, false, false), + ALICE_REFUNDING_A(105, true, true), + ALICE_REFUNDED(110, false, false); + + private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); + + public final int value; + public final boolean requiresAtData; + public final boolean requiresTradeData; + + State(int value, boolean requiresAtData, boolean requiresTradeData) { + this.value = value; + this.requiresAtData = requiresAtData; + this.requiresTradeData = requiresTradeData; + } + + public static State valueOf(int value) { + return map.get(value); + } + + @Override + public String getState() { + return this.name(); + } + + @Override + public int getStateValue() { + return this.value; + } + } + + /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ + private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms + + private static LitecoinACCTv1TradeBot instance; + + private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream() + .map(State::name) + .collect(Collectors.toUnmodifiableList()); + + private LitecoinACCTv1TradeBot() { + } + + public static synchronized LitecoinACCTv1TradeBot getInstance() { + if (instance == null) + instance = new LitecoinACCTv1TradeBot(); + + return instance; + } + + @Override + public List getEndStates() { + return this.endStates; + } + + /** + * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for LTC. + *

+ * Generates: + *

    + *
  • new 'trade' private key
    • + *
+ * Derives: + *
    + *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • + *
  • 'foreign' (as in Litecoin) public key, public key hash
    • + *
+ * A Qortal AT is then constructed including the following as constants in the 'data segment': + *
    + *
  • 'native'/Qortal 'trade' address - used as a MESSAGE contact
    • + *
  • 'foreign'/Litecoin public key hash - used by Alice's P2SH scripts to allow redeem
    • + *
  • QORT amount on offer by Bob
    • + *
  • LTC amount expected in return by Bob (from Alice)
    • + *
  • trading timeout, in case things go wrong and everyone needs to refund
    • + *
+ * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. + *

+ * Trade-bot will wait for Bob's AT to be deployed before taking next step. + *

+ * @param repository + * @param tradeBotCreateRequest + * @return raw, unsigned DEPLOY_AT transaction + * @throws DataException + */ + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + + // Convert Litecoin receiving address into public key hash (we only support P2PKH at this time) + Address litecoinReceivingAddress; + try { + litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); + } catch (AddressFormatException e) { + throw new DataException("Unsupported Litecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + } + if (litecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) + throw new DataException("Unsupported Litecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + + byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash(); + + PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); + + // Deploy AT + long timestamp = NTP.getTime(); + byte[] reference = creator.getLastReference(); + long fee = 0L; + byte[] signature = null; + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); + + String name = "QORT/LTC ACCT"; + String description = "QORT/LTC cross-chain trade"; + String aTType = "ACCT"; + String tags = "ACCT QORT LTC"; + byte[] creationBytes = LitecoinACCTv1.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount, + tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); + long amount = tradeBotCreateRequest.fundingQortAmount; + + DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + DeployAtTransaction.ensureATAddress(deployAtTransactionData); + String atAddress = deployAtTransactionData.getAtAddress(); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv1.NAME, + State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, + creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + null, null, + SupportedBlockchain.LITECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + tradeBotCreateRequest.foreignAmount, null, null, null, litecoinReceivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); + + // Attempt to backup the trade bot data + TradeBot.backupTradeBotData(repository, null); + + // Return to user for signing and broadcast as we don't have their Qortal private key + try { + return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); + } catch (TransformationException e) { + throw new DataException("Failed to transform DEPLOY_AT transaction?", e); + } + } + + /** + * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching LTC to an existing offer. + *

+ * Requires a chosen trade offer from Bob, passed by crossChainTradeData + * and access to a Litecoin wallet via xprv58. + *

+ * The crossChainTradeData contains the current trade offer state + * as extracted from the AT's data segment. + *

+ * Access to a funded wallet is via a Litecoin BIP32 hierarchical deterministic key, + * passed via xprv58. + * This key will be stored in your node's database + * to allow trade-bot to create/fund the necessary P2SH transactions! + * However, due to the nature of BIP32 keys, it is possible to give the trade-bot + * only a subset of wallet access (see BIP32 for more details). + *

+ * As an example, the xprv58 can be extract from a legacy, password-less + * Electrum wallet by going to the console tab and entering:
+ * wallet.keystore.xprv
+ * which should result in a base58 string starting with either 'xprv' (for Litecoin main-net) + * or 'tprv' for (Litecoin test-net). + *

+ * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. + *

+ * If sufficient funds are available, this method will actually fund the P2SH-A + * with the Litecoin amount expected by 'Bob'. + *

+ * If the Litecoin transaction is successfully broadcast to the network then + * we also send a MESSAGE to Bob's trade-bot to let them know. + *

+ * The trade-bot entry is saved to the repository and the cross-chain trading process commences. + *

+ * @param repository + * @param crossChainTradeData chosen trade OFFER that Alice wants to match + * @param xprv58 funded wallet xprv in base58 + * @return true if P2SH-A funding transaction successfully broadcast to Litecoin network, false otherwise + * @throws DataException + */ + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + byte[] secretA = TradeBot.generateSecret(); + byte[] hashOfSecretA = Crypto.hash160(secretA); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH + + // We need to generate lockTime-A: add tradeTimeout to now + long now = NTP.getTime(); + int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv1.NAME, + State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value, + receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secretA, hashOfSecretA, + SupportedBlockchain.LITECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); + + // Attempt to backup the trade bot data + // Include tradeBotData as an additional parameter, since it's not in the repository yet + TradeBot.backupTradeBotData(repository, Arrays.asList(tradeBotData)); + + // Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount + long p2shFee; + try { + p2shFee = Litecoin.getInstance().getP2shFee(now); + } catch (ForeignBlockchainException e) { + LOGGER.debug("Couldn't estimate Litecoin fees?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Fee for redeem/refund is subtracted from P2SH-A balance. + // Do not include fee for funding transaction as this is covered by buildSpend() + long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/; + + // P2SH-A to be funded + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); + String p2shAddress = Litecoin.getInstance().deriveP2shAddress(redeemScriptBytes); + + // Build transaction for funding P2SH-A + Transaction p2shFundingTransaction = Litecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); + if (p2shFundingTransaction == null) { + LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); + return ResponseResult.BALANCE_ISSUE; + } + + try { + Litecoin.getInstance().broadcastTransaction(p2shFundingTransaction); + } catch (ForeignBlockchainException e) { + // We couldn't fund P2SH-A at this time + LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Attempt to send MESSAGE to Bob's Qortal trade address + byte[] messageData = LitecoinACCTv1.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); + String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); + return ResponseResult.NETWORK_ISSUE; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress)); + + return ResponseResult.OK; + } + + @Override + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) + return true; + + // If the AT doesn't exist then we might as well let the user tidy up + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) + return true; + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + case ALICE_DONE: + case BOB_DONE: + case ALICE_REFUNDED: + case BOB_REFUNDED: + case ALICE_REFUNDING_A: + return true; + + default: + return false; + } + } + + @Override + public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) { + LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); + return; + } + + ATData atData = null; + CrossChainTradeData tradeData = null; + + if (tradeBotState.requiresAtData) { + // Attempt to fetch AT data + atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); + if (atData == null) { + LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + return; + } + + if (tradeBotState.requiresTradeData) { + tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + if (tradeData == null) { + LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); + return; + } + } + } + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + handleBobWaitingForAtConfirm(repository, tradeBotData); + break; + + case BOB_WAITING_FOR_MESSAGE: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_WAITING_FOR_AT_LOCK: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_AT_REDEEM: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_DONE: + case BOB_DONE: + break; + + case ALICE_REFUNDING_A: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_REFUNDED: + case BOB_REFUNDED: + break; + } + } + + /** + * Trade-bot is waiting for Bob's AT to deploy. + *

+ * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. + */ + private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { + if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) + return; + + // We've waited ages for AT to be confirmed into a block but something has gone awry. + // After this long we assume transaction loss so give up with trade-bot entry too. + tradeBotData.setState(State.BOB_REFUNDED.name()); + tradeBotData.setStateValue(State.BOB_REFUNDED.value); + tradeBotData.setTimestamp(NTP.getTime()); + // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + repository.saveChanges(); + + LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); + TradeBot.notifyStateChange(tradeBotData); + return; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, + () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. + *

+ * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, + * in which case trade-bot is done with this specific trade and finalizes on refunded state. + *

+ * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. + *

+ * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. + *

+ * Assuming P2SH-A has at least expected Litecoin balance, + * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. + *

+ * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. + *

+ * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to + * extract secret-A needed to redeem Alice's P2SH. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // If AT has finished then Bob likely cancelled his trade offer + if (atData.getIsFinished()) { + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); + return; + } + + Litecoin litecoin = Litecoin.getInstance(); + + String address = tradeBotData.getTradeNativeAddress(); + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); + + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + if (messageTransactionData.isText()) + continue; + + // We're expecting: HASH160(secret-A), Alice's Litecoin pubkeyhash and lockTime-A + byte[] messageData = messageTransactionData.getData(); + LitecoinACCTv1.OfferMessageData offerMessageData = LitecoinACCTv1.extractOfferMessageData(messageData); + if (offerMessageData == null) + continue; + + byte[] aliceForeignPublicKeyHash = offerMessageData.partnerLitecoinPKH; + byte[] hashOfSecretA = offerMessageData.hashOfSecretA; + int lockTimeA = (int) offerMessageData.lockTimeA; + long messageTimestamp = messageTransactionData.getTimestamp(); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(messageTimestamp, lockTimeA); + + // Determine P2SH-A address and confirm funded + byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // There might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // We've already redeemed this? + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case FUNDED: + // Fall-through out of switch... + break; + } + + // Good to go - send MESSAGE to AT + + String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); + + // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume + byte[] outgoingMessageData = LitecoinACCTv1.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); + + outgoingMessageTransaction.computeNonce(); + outgoingMessageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, + () -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress)); + + return; + } + } + + /** + * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. + *

+ * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow + * this process has taken so long that we've reached P2SH-A's locktime, or that someone else + * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. + *

+ * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. + *

+ * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice. + *

+ * In revealing a valid secret-A, Bob can then redeem the LTC funds from P2SH-A. + *

+ * @throws ForeignBlockchainException + */ + private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Litecoin litecoin = Litecoin.getInstance(); + int lockTimeA = tradeBotData.getLockTimeA(); + + // Refund P2SH-A if we've passed lockTime-A + if (NTP.getTime() >= lockTimeA * 1000L) { + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + case FUNDED: + break; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Already redeemed? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); + return; + + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> atData.getIsFinished() + ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) + : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); + + return; + } + + // We're waiting for AT to be in TRADE mode + if (crossChainTradeData.mode != AcctMode.TRADING) + return; + + // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above + + // Find our MESSAGE to AT from previous state + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), + crossChainTradeData.qortalCreatorTradeAddress, null, null, null); + if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { + LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); + return; + } + + long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(recipientMessageTimestamp, lockTimeA); + + // Our calculated refundTimeout should match AT's refundTimeout + if (refundTimeout != crossChainTradeData.refundTimeout) { + LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout)); + // We'll eventually refund + return; + } + + // We're good to redeem AT + + // Send 'redeem' MESSAGE to AT using both secret + byte[] secretA = tradeBotData.getSecret(); + String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH + byte[] messageData = LitecoinACCTv1.buildRedeemMessage(secretA, qortalReceivingAddress); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // Reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("Redeeming AT %s. Funds should arrive at %s", + tradeBotData.getAtAddress(), qortalReceivingAddress)); + } + + /** + * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the LTC funds from P2SH-A. + *

+ * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, + * trade-bot is done with this specific trade and finalizes in refunded state. + *

+ * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the LTC funds from P2SH-A + * to Bob's 'foreign'/Litecoin trade legacy-format address, as derived from trade private key. + *

+ * (This could potentially be 'improved' to send LTC to any address of Bob's choosing by changing the transaction output). + *

+ * If trade-bot successfully broadcasts the transaction, then this specific trade is done. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // AT should be 'finished' once Alice has redeemed QORT funds + if (!atData.getIsFinished()) + // Not finished yet + return; + + // If AT is REFUNDED or CANCELLED then something has gone wrong + if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) { + // Alice hasn't redeemed the QORT, so there is no point in trying to redeem the LTC + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); + + return; + } + + byte[] secretA = LitecoinACCTv1.getInstance().findSecretA(repository, crossChainTradeData); + if (secretA == null) { + LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); + return; + } + + // Use secret-A to redeem P2SH-A + + Litecoin litecoin = Litecoin.getInstance(); + + byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + int lockTimeA = crossChainTradeData.lockTimeA; + byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Double-check that we have redeemed P2SH-A... + break; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // Wait for AT to auto-refund + return; + + case FUNDED: { + Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(litecoin.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); + + litecoin.broadcastTransaction(p2shRedeemTransaction); + break; + } + } + + String receivingAddress = litecoin.pkhToAddress(receivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); + } + + /** + * Trade-bot is attempting to refund P2SH-A. + * @throws ForeignBlockchainException + */ + private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + int lockTimeA = tradeBotData.getLockTimeA(); + + // We can't refund P2SH-A until lockTime-A has passed + if (NTP.getTime() <= lockTimeA * 1000L) + return; + + Litecoin litecoin = Litecoin.getInstance(); + + // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) + int medianBlockTime = litecoin.getMedianBlockTime(); + if (medianBlockTime <= lockTimeA) + return; + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-A to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Too late! + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent!", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + break; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); + + // Determine receive address for refund + String receiveAddress = litecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + Address receiving = Address.fromString(litecoin.getNetworkParameters(), receiveAddress); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(litecoin.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); + + litecoin.broadcastTransaction(p2shRefundTransaction); + break; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); + } + + /** + * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. + *

+ * Will automatically update trade-bot state to ALICE_REFUNDING_A or ALICE_DONE as necessary. + * + * @throws DataException + * @throws ForeignBlockchainException + */ + private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // This is OK + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) + return false; + + boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); + + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING) + if (isAtLockedToUs) { + // AT is trading with us - OK + return false; + } else { + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress)); + + return true; + } + + if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { + // We've redeemed already? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); + } else { + // Any other state is not good, so start defensive refund + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); + } + + return true; + } + + private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { + return (lockTimeA - tradeTimeout * 60) * 1000L; + } + +} diff --git a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv2TradeBot.java b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv2TradeBot.java new file mode 100644 index 000000000..6261339ac --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv2TradeBot.java @@ -0,0 +1,885 @@ +package org.qortal.controller.tradebot; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.*; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.asset.Asset; +import org.qortal.crosschain.*; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.DeployAtTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +/** + * Performing cross-chain trading steps on behalf of user. + *

+ * We deal with three different independent state-spaces here: + *

    + *
  • Qortal blockchain
    • + *
  • Foreign blockchain
    • + *
  • Trade-bot entries
    • + *
+ */ +public class LitecoinACCTv2TradeBot implements AcctTradeBot { + + private static final Logger LOGGER = LogManager.getLogger(LitecoinACCTv2TradeBot.class); + + public enum State implements TradeBot.StateNameAndValueSupplier { + BOB_WAITING_FOR_AT_CONFIRM(10, false, false), + BOB_WAITING_FOR_MESSAGE(15, true, true), + BOB_WAITING_FOR_AT_REDEEM(25, true, true), + BOB_DONE(30, false, false), + BOB_REFUNDED(35, false, false), + + ALICE_WAITING_FOR_AT_LOCK(85, true, true), + ALICE_DONE(95, false, false), + ALICE_REFUNDING_A(105, true, true), + ALICE_REFUNDED(110, false, false); + + private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); + + public final int value; + public final boolean requiresAtData; + public final boolean requiresTradeData; + + State(int value, boolean requiresAtData, boolean requiresTradeData) { + this.value = value; + this.requiresAtData = requiresAtData; + this.requiresTradeData = requiresTradeData; + } + + public static State valueOf(int value) { + return map.get(value); + } + + @Override + public String getState() { + return this.name(); + } + + @Override + public int getStateValue() { + return this.value; + } + } + + /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ + private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms + + private static LitecoinACCTv2TradeBot instance; + + private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream() + .map(State::name) + .collect(Collectors.toUnmodifiableList()); + + private LitecoinACCTv2TradeBot() { + } + + public static synchronized LitecoinACCTv2TradeBot getInstance() { + if (instance == null) + instance = new LitecoinACCTv2TradeBot(); + + return instance; + } + + @Override + public List getEndStates() { + return this.endStates; + } + + /** + * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for LTC. + *

+ * Generates: + *

    + *
  • new 'trade' private key
    • + *
+ * Derives: + *
    + *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • + *
  • 'foreign' (as in Litecoin) public key, public key hash
    • + *
+ * A Qortal AT is then constructed including the following as constants in the 'data segment': + *
    + *
  • 'native'/Qortal 'trade' address - used as a MESSAGE contact
    • + *
  • 'foreign'/Litecoin public key hash - used by Alice's P2SH scripts to allow redeem
    • + *
  • QORT amount on offer by Bob
    • + *
  • LTC amount expected in return by Bob (from Alice)
    • + *
  • trading timeout, in case things go wrong and everyone needs to refund
    • + *
+ * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. + *

+ * Trade-bot will wait for Bob's AT to be deployed before taking next step. + *

+ * @param repository + * @param tradeBotCreateRequest + * @return raw, unsigned DEPLOY_AT transaction + * @throws DataException + */ + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + + // Convert Litecoin receiving address into public key hash (we only support P2PKH at this time) + Address litecoinReceivingAddress; + try { + litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); + } catch (AddressFormatException e) { + throw new DataException("Unsupported Litecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + } + if (litecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) + throw new DataException("Unsupported Litecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + + byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash(); + + PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); + + // Deploy AT + long timestamp = NTP.getTime(); + byte[] reference = creator.getLastReference(); + long fee = 0L; + byte[] signature = null; + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); + + String name = "QORT/LTC ACCT"; + String description = "QORT/LTC cross-chain trade"; + String aTType = "ACCT"; + String tags = "ACCT QORT LTC"; + byte[] creationBytes = LitecoinACCTv2.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount, + tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); + long amount = tradeBotCreateRequest.fundingQortAmount; + + DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + DeployAtTransaction.ensureATAddress(deployAtTransactionData); + String atAddress = deployAtTransactionData.getAtAddress(); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv2.NAME, + State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, + creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + null, null, + SupportedBlockchain.LITECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + tradeBotCreateRequest.foreignAmount, null, null, null, litecoinReceivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); + + // Attempt to backup the trade bot data + TradeBot.backupTradeBotData(repository, null); + + // Return to user for signing and broadcast as we don't have their Qortal private key + try { + return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); + } catch (TransformationException e) { + throw new DataException("Failed to transform DEPLOY_AT transaction?", e); + } + } + + /** + * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching LTC to an existing offer. + *

+ * Requires a chosen trade offer from Bob, passed by crossChainTradeData + * and access to a Litecoin wallet via xprv58. + *

+ * The crossChainTradeData contains the current trade offer state + * as extracted from the AT's data segment. + *

+ * Access to a funded wallet is via a Litecoin BIP32 hierarchical deterministic key, + * passed via xprv58. + * This key will be stored in your node's database + * to allow trade-bot to create/fund the necessary P2SH transactions! + * However, due to the nature of BIP32 keys, it is possible to give the trade-bot + * only a subset of wallet access (see BIP32 for more details). + *

+ * As an example, the xprv58 can be extract from a legacy, password-less + * Electrum wallet by going to the console tab and entering:
+ * wallet.keystore.xprv
+ * which should result in a base58 string starting with either 'xprv' (for Litecoin main-net) + * or 'tprv' for (Litecoin test-net). + *

+ * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. + *

+ * If sufficient funds are available, this method will actually fund the P2SH-A + * with the Litecoin amount expected by 'Bob'. + *

+ * If the Litecoin transaction is successfully broadcast to the network then + * we also send a MESSAGE to Bob's trade-bot to let them know. + *

+ * The trade-bot entry is saved to the repository and the cross-chain trading process commences. + *

+ * @param repository + * @param crossChainTradeData chosen trade OFFER that Alice wants to match + * @param xprv58 funded wallet xprv in base58 + * @return true if P2SH-A funding transaction successfully broadcast to Litecoin network, false otherwise + * @throws DataException + */ + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + byte[] secretA = TradeBot.generateSecret(); + byte[] hashOfSecretA = Crypto.hash160(secretA); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH + + // We need to generate lockTime-A: add tradeTimeout to now + long now = NTP.getTime(); + int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv2.NAME, + State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value, + receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secretA, hashOfSecretA, + SupportedBlockchain.LITECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); + + // Attempt to backup the trade bot data + // Include tradeBotData as an additional parameter, since it's not in the repository yet + TradeBot.backupTradeBotData(repository, Arrays.asList(tradeBotData)); + + // Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount + long p2shFee; + try { + p2shFee = Litecoin.getInstance().getP2shFee(now); + } catch (ForeignBlockchainException e) { + LOGGER.debug("Couldn't estimate Litecoin fees?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Fee for redeem/refund is subtracted from P2SH-A balance. + // Do not include fee for funding transaction as this is covered by buildSpend() + long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/; + + // P2SH-A to be funded + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); + String p2shAddress = Litecoin.getInstance().deriveP2shAddress(redeemScriptBytes); + + // Build transaction for funding P2SH-A + Transaction p2shFundingTransaction = Litecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); + if (p2shFundingTransaction == null) { + LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); + return ResponseResult.BALANCE_ISSUE; + } + + try { + Litecoin.getInstance().broadcastTransaction(p2shFundingTransaction); + } catch (ForeignBlockchainException e) { + // We couldn't fund P2SH-A at this time + LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Attempt to send MESSAGE to Bob's Qortal trade address + byte[] messageData = LitecoinACCTv2.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); + String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); + return ResponseResult.NETWORK_ISSUE; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress)); + + return ResponseResult.OK; + } + + @Override + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) + return true; + + // If the AT doesn't exist then we might as well let the user tidy up + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) + return true; + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + case ALICE_DONE: + case BOB_DONE: + case ALICE_REFUNDED: + case BOB_REFUNDED: + case ALICE_REFUNDING_A: + return true; + + default: + return false; + } + } + + @Override + public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) { + LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); + return; + } + + ATData atData = null; + CrossChainTradeData tradeData = null; + + if (tradeBotState.requiresAtData) { + // Attempt to fetch AT data + atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); + if (atData == null) { + LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + return; + } + + if (tradeBotState.requiresTradeData) { + tradeData = LitecoinACCTv2.getInstance().populateTradeData(repository, atData); + if (tradeData == null) { + LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); + return; + } + } + } + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + handleBobWaitingForAtConfirm(repository, tradeBotData); + break; + + case BOB_WAITING_FOR_MESSAGE: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_WAITING_FOR_AT_LOCK: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_AT_REDEEM: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_DONE: + case BOB_DONE: + break; + + case ALICE_REFUNDING_A: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_REFUNDED: + case BOB_REFUNDED: + break; + } + } + + /** + * Trade-bot is waiting for Bob's AT to deploy. + *

+ * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. + */ + private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { + if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) + return; + + // We've waited ages for AT to be confirmed into a block but something has gone awry. + // After this long we assume transaction loss so give up with trade-bot entry too. + tradeBotData.setState(State.BOB_REFUNDED.name()); + tradeBotData.setStateValue(State.BOB_REFUNDED.value); + tradeBotData.setTimestamp(NTP.getTime()); + // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + repository.saveChanges(); + + LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); + TradeBot.notifyStateChange(tradeBotData); + return; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, + () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. + *

+ * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, + * in which case trade-bot is done with this specific trade and finalizes on refunded state. + *

+ * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. + *

+ * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. + *

+ * Assuming P2SH-A has at least expected Litecoin balance, + * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. + *

+ * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. + *

+ * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to + * extract secret-A needed to redeem Alice's P2SH. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // If AT has finished then Bob likely cancelled his trade offer + if (atData.getIsFinished()) { + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); + return; + } + + Litecoin litecoin = Litecoin.getInstance(); + + String address = tradeBotData.getTradeNativeAddress(); + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); + + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + if (messageTransactionData.isText()) + continue; + + // We're expecting: HASH160(secret-A), Alice's Litecoin pubkeyhash and lockTime-A + byte[] messageData = messageTransactionData.getData(); + LitecoinACCTv2.OfferMessageData offerMessageData = LitecoinACCTv2.extractOfferMessageData(messageData); + if (offerMessageData == null) + continue; + + byte[] aliceForeignPublicKeyHash = offerMessageData.partnerLitecoinPKH; + byte[] hashOfSecretA = offerMessageData.hashOfSecretA; + int lockTimeA = (int) offerMessageData.lockTimeA; + long messageTimestamp = messageTransactionData.getTimestamp(); + int refundTimeout = LitecoinACCTv2.calcRefundTimeout(messageTimestamp, lockTimeA); + + // Determine P2SH-A address and confirm funded + byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // There might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // We've already redeemed this? + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case FUNDED: + // Fall-through out of switch... + break; + } + + // Good to go - send MESSAGE to AT + + String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); + + // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume + byte[] outgoingMessageData = LitecoinACCTv2.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); + + outgoingMessageTransaction.computeNonce(); + outgoingMessageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, + () -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress)); + + return; + } + } + + /** + * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. + *

+ * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow + * this process has taken so long that we've reached P2SH-A's locktime, or that someone else + * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. + *

+ * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. + *

+ * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice. + *

+ * In revealing a valid secret-A, Bob can then redeem the LTC funds from P2SH-A. + *

+ * @throws ForeignBlockchainException + */ + private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Litecoin litecoin = Litecoin.getInstance(); + int lockTimeA = tradeBotData.getLockTimeA(); + + // Refund P2SH-A if we've passed lockTime-A + if (NTP.getTime() >= lockTimeA * 1000L) { + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + case FUNDED: + break; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Already redeemed? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); + return; + + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> atData.getIsFinished() + ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) + : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); + + return; + } + + // We're waiting for AT to be in TRADE mode + if (crossChainTradeData.mode != AcctMode.TRADING) + return; + + // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above + + // Find our MESSAGE to AT from previous state + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), + crossChainTradeData.qortalCreatorTradeAddress, null, null, null); + if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { + LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); + return; + } + + long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); + int refundTimeout = LitecoinACCTv2.calcRefundTimeout(recipientMessageTimestamp, lockTimeA); + + // Our calculated refundTimeout should match AT's refundTimeout + if (refundTimeout != crossChainTradeData.refundTimeout) { + LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout)); + // We'll eventually refund + return; + } + + // We're good to redeem AT + + // Send 'redeem' MESSAGE to AT using both secret + byte[] secretA = tradeBotData.getSecret(); + String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH + byte[] messageData = LitecoinACCTv2.buildRedeemMessage(secretA, qortalReceivingAddress); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // Reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("Redeeming AT %s. Funds should arrive at %s", + tradeBotData.getAtAddress(), qortalReceivingAddress)); + } + + /** + * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the LTC funds from P2SH-A. + *

+ * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, + * trade-bot is done with this specific trade and finalizes in refunded state. + *

+ * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the LTC funds from P2SH-A + * to Bob's 'foreign'/Litecoin trade legacy-format address, as derived from trade private key. + *

+ * (This could potentially be 'improved' to send LTC to any address of Bob's choosing by changing the transaction output). + *

+ * If trade-bot successfully broadcasts the transaction, then this specific trade is done. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // AT should be 'finished' once Alice has redeemed QORT funds + if (!atData.getIsFinished()) + // Not finished yet + return; + + // If AT is REFUNDED or CANCELLED then something has gone wrong + if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) { + // Alice hasn't redeemed the QORT, so there is no point in trying to redeem the LTC + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); + + return; + } + + byte[] secretA = LitecoinACCTv2.getInstance().findSecretA(repository, crossChainTradeData); + if (secretA == null) { + LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); + return; + } + + // Use secret-A to redeem P2SH-A + + Litecoin litecoin = Litecoin.getInstance(); + + byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + int lockTimeA = crossChainTradeData.lockTimeA; + byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Double-check that we have redeemed P2SH-A... + break; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // Wait for AT to auto-refund + return; + + case FUNDED: { + Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(litecoin.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); + + litecoin.broadcastTransaction(p2shRedeemTransaction); + break; + } + } + + String receivingAddress = litecoin.pkhToAddress(receivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); + } + + /** + * Trade-bot is attempting to refund P2SH-A. + * @throws ForeignBlockchainException + */ + private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + int lockTimeA = tradeBotData.getLockTimeA(); + + // We can't refund P2SH-A until lockTime-A has passed + if (NTP.getTime() <= lockTimeA * 1000L) + return; + + Litecoin litecoin = Litecoin.getInstance(); + + // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) + int medianBlockTime = litecoin.getMedianBlockTime(); + if (medianBlockTime <= lockTimeA) + return; + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-A to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Too late! + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent!", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + break; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); + + // Determine receive address for refund + String receiveAddress = litecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + Address receiving = Address.fromString(litecoin.getNetworkParameters(), receiveAddress); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(litecoin.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); + + litecoin.broadcastTransaction(p2shRefundTransaction); + break; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); + } + + /** + * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. + *

+ * Will automatically update trade-bot state to ALICE_REFUNDING_A or ALICE_DONE as necessary. + * + * @throws DataException + * @throws ForeignBlockchainException + */ + private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // This is OK + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) + return false; + + boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); + + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING) + if (isAtLockedToUs) { + // AT is trading with us - OK + return false; + } else { + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress)); + + return true; + } + + if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { + // We've redeemed already? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); + } else { + // Any other state is not good, so start defensive refund + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); + } + + return true; + } + + private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { + return (lockTimeA - tradeTimeout * 60) * 1000L; + } + +} diff --git a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv3TradeBot.java b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv3TradeBot.java new file mode 100644 index 000000000..a31a1a28a --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv3TradeBot.java @@ -0,0 +1,885 @@ +package org.qortal.controller.tradebot; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.*; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.asset.Asset; +import org.qortal.crosschain.*; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.DeployAtTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +/** + * Performing cross-chain trading steps on behalf of user. + *

+ * We deal with three different independent state-spaces here: + *

    + *
  • Qortal blockchain
    • + *
  • Foreign blockchain
    • + *
  • Trade-bot entries
    • + *
+ */ +public class LitecoinACCTv3TradeBot implements AcctTradeBot { + + private static final Logger LOGGER = LogManager.getLogger(LitecoinACCTv3TradeBot.class); + + public enum State implements TradeBot.StateNameAndValueSupplier { + BOB_WAITING_FOR_AT_CONFIRM(10, false, false), + BOB_WAITING_FOR_MESSAGE(15, true, true), + BOB_WAITING_FOR_AT_REDEEM(25, true, true), + BOB_DONE(30, false, false), + BOB_REFUNDED(35, false, false), + + ALICE_WAITING_FOR_AT_LOCK(85, true, true), + ALICE_DONE(95, false, false), + ALICE_REFUNDING_A(105, true, true), + ALICE_REFUNDED(110, false, false); + + private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); + + public final int value; + public final boolean requiresAtData; + public final boolean requiresTradeData; + + State(int value, boolean requiresAtData, boolean requiresTradeData) { + this.value = value; + this.requiresAtData = requiresAtData; + this.requiresTradeData = requiresTradeData; + } + + public static State valueOf(int value) { + return map.get(value); + } + + @Override + public String getState() { + return this.name(); + } + + @Override + public int getStateValue() { + return this.value; + } + } + + /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ + private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms + + private static LitecoinACCTv3TradeBot instance; + + private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream() + .map(State::name) + .collect(Collectors.toUnmodifiableList()); + + private LitecoinACCTv3TradeBot() { + } + + public static synchronized LitecoinACCTv3TradeBot getInstance() { + if (instance == null) + instance = new LitecoinACCTv3TradeBot(); + + return instance; + } + + @Override + public List getEndStates() { + return this.endStates; + } + + /** + * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for LTC. + *

+ * Generates: + *

    + *
  • new 'trade' private key
    • + *
+ * Derives: + *
    + *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • + *
  • 'foreign' (as in Litecoin) public key, public key hash
    • + *
+ * A Qortal AT is then constructed including the following as constants in the 'data segment': + *
    + *
  • 'native'/Qortal 'trade' address - used as a MESSAGE contact
    • + *
  • 'foreign'/Litecoin public key hash - used by Alice's P2SH scripts to allow redeem
    • + *
  • QORT amount on offer by Bob
    • + *
  • LTC amount expected in return by Bob (from Alice)
    • + *
  • trading timeout, in case things go wrong and everyone needs to refund
    • + *
+ * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. + *

+ * Trade-bot will wait for Bob's AT to be deployed before taking next step. + *

+ * @param repository + * @param tradeBotCreateRequest + * @return raw, unsigned DEPLOY_AT transaction + * @throws DataException + */ + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + + // Convert Litecoin receiving address into public key hash (we only support P2PKH at this time) + Address litecoinReceivingAddress; + try { + litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); + } catch (AddressFormatException e) { + throw new DataException("Unsupported Litecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + } + if (litecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) + throw new DataException("Unsupported Litecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + + byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash(); + + PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); + + // Deploy AT + long timestamp = NTP.getTime(); + byte[] reference = creator.getLastReference(); + long fee = 0L; + byte[] signature = null; + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); + + String name = "QORT/LTC ACCT"; + String description = "QORT/LTC cross-chain trade"; + String aTType = "ACCT"; + String tags = "ACCT QORT LTC"; + byte[] creationBytes = LitecoinACCTv3.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount, + tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); + long amount = tradeBotCreateRequest.fundingQortAmount; + + DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + DeployAtTransaction.ensureATAddress(deployAtTransactionData); + String atAddress = deployAtTransactionData.getAtAddress(); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv3.NAME, + State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, + creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + null, null, + SupportedBlockchain.LITECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + tradeBotCreateRequest.foreignAmount, null, null, null, litecoinReceivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); + + // Attempt to backup the trade bot data + TradeBot.backupTradeBotData(repository, null); + + // Return to user for signing and broadcast as we don't have their Qortal private key + try { + return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); + } catch (TransformationException e) { + throw new DataException("Failed to transform DEPLOY_AT transaction?", e); + } + } + + /** + * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching LTC to an existing offer. + *

+ * Requires a chosen trade offer from Bob, passed by crossChainTradeData + * and access to a Litecoin wallet via xprv58. + *

+ * The crossChainTradeData contains the current trade offer state + * as extracted from the AT's data segment. + *

+ * Access to a funded wallet is via a Litecoin BIP32 hierarchical deterministic key, + * passed via xprv58. + * This key will be stored in your node's database + * to allow trade-bot to create/fund the necessary P2SH transactions! + * However, due to the nature of BIP32 keys, it is possible to give the trade-bot + * only a subset of wallet access (see BIP32 for more details). + *

+ * As an example, the xprv58 can be extract from a legacy, password-less + * Electrum wallet by going to the console tab and entering:
+ * wallet.keystore.xprv
+ * which should result in a base58 string starting with either 'xprv' (for Litecoin main-net) + * or 'tprv' for (Litecoin test-net). + *

+ * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. + *

+ * If sufficient funds are available, this method will actually fund the P2SH-A + * with the Litecoin amount expected by 'Bob'. + *

+ * If the Litecoin transaction is successfully broadcast to the network then + * we also send a MESSAGE to Bob's trade-bot to let them know. + *

+ * The trade-bot entry is saved to the repository and the cross-chain trading process commences. + *

+ * @param repository + * @param crossChainTradeData chosen trade OFFER that Alice wants to match + * @param xprv58 funded wallet xprv in base58 + * @return true if P2SH-A funding transaction successfully broadcast to Litecoin network, false otherwise + * @throws DataException + */ + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + byte[] secretA = TradeBot.generateSecret(); + byte[] hashOfSecretA = Crypto.hash160(secretA); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH + + // We need to generate lockTime-A: add tradeTimeout to now + long now = NTP.getTime(); + int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv3.NAME, + State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value, + receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secretA, hashOfSecretA, + SupportedBlockchain.LITECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); + + // Attempt to backup the trade bot data + // Include tradeBotData as an additional parameter, since it's not in the repository yet + TradeBot.backupTradeBotData(repository, Arrays.asList(tradeBotData)); + + // Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount + long p2shFee; + try { + p2shFee = Litecoin.getInstance().getP2shFee(now); + } catch (ForeignBlockchainException e) { + LOGGER.debug("Couldn't estimate Litecoin fees?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Fee for redeem/refund is subtracted from P2SH-A balance. + // Do not include fee for funding transaction as this is covered by buildSpend() + long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/; + + // P2SH-A to be funded + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); + String p2shAddress = Litecoin.getInstance().deriveP2shAddress(redeemScriptBytes); + + // Build transaction for funding P2SH-A + Transaction p2shFundingTransaction = Litecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); + if (p2shFundingTransaction == null) { + LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); + return ResponseResult.BALANCE_ISSUE; + } + + try { + Litecoin.getInstance().broadcastTransaction(p2shFundingTransaction); + } catch (ForeignBlockchainException e) { + // We couldn't fund P2SH-A at this time + LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Attempt to send MESSAGE to Bob's Qortal trade address + byte[] messageData = LitecoinACCTv3.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); + String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); + return ResponseResult.NETWORK_ISSUE; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress)); + + return ResponseResult.OK; + } + + @Override + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) + return true; + + // If the AT doesn't exist then we might as well let the user tidy up + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) + return true; + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + case ALICE_DONE: + case BOB_DONE: + case ALICE_REFUNDED: + case BOB_REFUNDED: + case ALICE_REFUNDING_A: + return true; + + default: + return false; + } + } + + @Override + public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) { + LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); + return; + } + + ATData atData = null; + CrossChainTradeData tradeData = null; + + if (tradeBotState.requiresAtData) { + // Attempt to fetch AT data + atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); + if (atData == null) { + LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + return; + } + + if (tradeBotState.requiresTradeData) { + tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData); + if (tradeData == null) { + LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); + return; + } + } + } + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + handleBobWaitingForAtConfirm(repository, tradeBotData); + break; + + case BOB_WAITING_FOR_MESSAGE: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_WAITING_FOR_AT_LOCK: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_AT_REDEEM: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_DONE: + case BOB_DONE: + break; + + case ALICE_REFUNDING_A: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_REFUNDED: + case BOB_REFUNDED: + break; + } + } + + /** + * Trade-bot is waiting for Bob's AT to deploy. + *

+ * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. + */ + private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { + if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) + return; + + // We've waited ages for AT to be confirmed into a block but something has gone awry. + // After this long we assume transaction loss so give up with trade-bot entry too. + tradeBotData.setState(State.BOB_REFUNDED.name()); + tradeBotData.setStateValue(State.BOB_REFUNDED.value); + tradeBotData.setTimestamp(NTP.getTime()); + // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + repository.saveChanges(); + + LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); + TradeBot.notifyStateChange(tradeBotData); + return; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, + () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. + *

+ * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, + * in which case trade-bot is done with this specific trade and finalizes on refunded state. + *

+ * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. + *

+ * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. + *

+ * Assuming P2SH-A has at least expected Litecoin balance, + * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. + *

+ * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. + *

+ * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to + * extract secret-A needed to redeem Alice's P2SH. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // If AT has finished then Bob likely cancelled his trade offer + if (atData.getIsFinished()) { + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); + return; + } + + Litecoin litecoin = Litecoin.getInstance(); + + String address = tradeBotData.getTradeNativeAddress(); + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); + + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + if (messageTransactionData.isText()) + continue; + + // We're expecting: HASH160(secret-A), Alice's Litecoin pubkeyhash and lockTime-A + byte[] messageData = messageTransactionData.getData(); + LitecoinACCTv3.OfferMessageData offerMessageData = LitecoinACCTv3.extractOfferMessageData(messageData); + if (offerMessageData == null) + continue; + + byte[] aliceForeignPublicKeyHash = offerMessageData.partnerLitecoinPKH; + byte[] hashOfSecretA = offerMessageData.hashOfSecretA; + int lockTimeA = (int) offerMessageData.lockTimeA; + long messageTimestamp = messageTransactionData.getTimestamp(); + int refundTimeout = LitecoinACCTv3.calcRefundTimeout(messageTimestamp, lockTimeA); + + // Determine P2SH-A address and confirm funded + byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // There might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // We've already redeemed this? + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case FUNDED: + // Fall-through out of switch... + break; + } + + // Good to go - send MESSAGE to AT + + String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); + + // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume + byte[] outgoingMessageData = LitecoinACCTv3.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); + + outgoingMessageTransaction.computeNonce(); + outgoingMessageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, + () -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress)); + + return; + } + } + + /** + * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. + *

+ * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow + * this process has taken so long that we've reached P2SH-A's locktime, or that someone else + * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. + *

+ * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. + *

+ * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice. + *

+ * In revealing a valid secret-A, Bob can then redeem the LTC funds from P2SH-A. + *

+ * @throws ForeignBlockchainException + */ + private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Litecoin litecoin = Litecoin.getInstance(); + int lockTimeA = tradeBotData.getLockTimeA(); + + // Refund P2SH-A if we've passed lockTime-A + if (NTP.getTime() >= lockTimeA * 1000L) { + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + case FUNDED: + break; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Already redeemed? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); + return; + + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> atData.getIsFinished() + ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) + : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); + + return; + } + + // We're waiting for AT to be in TRADE mode + if (crossChainTradeData.mode != AcctMode.TRADING) + return; + + // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above + + // Find our MESSAGE to AT from previous state + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), + crossChainTradeData.qortalCreatorTradeAddress, null, null, null); + if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { + LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); + return; + } + + long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); + int refundTimeout = LitecoinACCTv3.calcRefundTimeout(recipientMessageTimestamp, lockTimeA); + + // Our calculated refundTimeout should match AT's refundTimeout + if (refundTimeout != crossChainTradeData.refundTimeout) { + LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout)); + // We'll eventually refund + return; + } + + // We're good to redeem AT + + // Send 'redeem' MESSAGE to AT using both secret + byte[] secretA = tradeBotData.getSecret(); + String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH + byte[] messageData = LitecoinACCTv3.buildRedeemMessage(secretA, qortalReceivingAddress); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // Reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("Redeeming AT %s. Funds should arrive at %s", + tradeBotData.getAtAddress(), qortalReceivingAddress)); + } + + /** + * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the LTC funds from P2SH-A. + *

+ * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, + * trade-bot is done with this specific trade and finalizes in refunded state. + *

+ * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the LTC funds from P2SH-A + * to Bob's 'foreign'/Litecoin trade legacy-format address, as derived from trade private key. + *

+ * (This could potentially be 'improved' to send LTC to any address of Bob's choosing by changing the transaction output). + *

+ * If trade-bot successfully broadcasts the transaction, then this specific trade is done. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // AT should be 'finished' once Alice has redeemed QORT funds + if (!atData.getIsFinished()) + // Not finished yet + return; + + // If AT is REFUNDED or CANCELLED then something has gone wrong + if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) { + // Alice hasn't redeemed the QORT, so there is no point in trying to redeem the LTC + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); + + return; + } + + byte[] secretA = LitecoinACCTv3.getInstance().findSecretA(repository, crossChainTradeData); + if (secretA == null) { + LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); + return; + } + + // Use secret-A to redeem P2SH-A + + Litecoin litecoin = Litecoin.getInstance(); + + byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + int lockTimeA = crossChainTradeData.lockTimeA; + byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Double-check that we have redeemed P2SH-A... + break; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // Wait for AT to auto-refund + return; + + case FUNDED: { + Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(litecoin.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); + + litecoin.broadcastTransaction(p2shRedeemTransaction); + break; + } + } + + String receivingAddress = litecoin.pkhToAddress(receivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); + } + + /** + * Trade-bot is attempting to refund P2SH-A. + * @throws ForeignBlockchainException + */ + private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + int lockTimeA = tradeBotData.getLockTimeA(); + + // We can't refund P2SH-A until lockTime-A has passed + if (NTP.getTime() <= lockTimeA * 1000L) + return; + + Litecoin litecoin = Litecoin.getInstance(); + + // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) + int medianBlockTime = litecoin.getMedianBlockTime(); + if (medianBlockTime <= lockTimeA) + return; + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-A to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Too late! + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent!", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + break; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); + + // Determine receive address for refund + String receiveAddress = litecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + Address receiving = Address.fromString(litecoin.getNetworkParameters(), receiveAddress); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(litecoin.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); + + litecoin.broadcastTransaction(p2shRefundTransaction); + break; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); + } + + /** + * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. + *

+ * Will automatically update trade-bot state to ALICE_REFUNDING_A or ALICE_DONE as necessary. + * + * @throws DataException + * @throws ForeignBlockchainException + */ + private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // This is OK + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) + return false; + + boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); + + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING) + if (isAtLockedToUs) { + // AT is trading with us - OK + return false; + } else { + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress)); + + return true; + } + + if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { + // We've redeemed already? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); + } else { + // Any other state is not good, so start defensive refund + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); + } + + return true; + } + + private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { + return (lockTimeA - tradeTimeout * 60) * 1000L; + } + +} diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java new file mode 100644 index 000000000..2f9c31212 --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -0,0 +1,375 @@ +package org.qortal.controller.tradebot; + +import java.awt.TrayIcon.MessageType; +import java.security.SecureRandom; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.util.Supplier; +import org.bitcoinj.core.ECKey; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.controller.Controller; +import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult; +import org.qortal.crosschain.*; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.Listener; +import org.qortal.group.Group; +import org.qortal.gui.SysTray; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBImportExport; +import org.qortal.settings.Settings; +import org.qortal.transaction.PresenceTransaction; +import org.qortal.transaction.PresenceTransaction.PresenceType; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.NTP; + +import com.google.common.primitives.Longs; + +/** + * Performing cross-chain trading steps on behalf of user. + *

+ * We deal with three different independent state-spaces here: + *

    + *
  • Qortal blockchain
    • + *
  • Foreign blockchain
    • + *
  • Trade-bot entries
    • + *
+ */ +public class TradeBot implements Listener { + + private static final Logger LOGGER = LogManager.getLogger(TradeBot.class); + private static final Random RANDOM = new SecureRandom(); + + public interface StateNameAndValueSupplier { + public String getState(); + public int getStateValue(); + } + + public static class StateChangeEvent implements Event { + private final TradeBotData tradeBotData; + + public StateChangeEvent(TradeBotData tradeBotData) { + this.tradeBotData = tradeBotData; + } + + public TradeBotData getTradeBotData() { + return this.tradeBotData; + } + } + + private static final Map, Supplier> acctTradeBotSuppliers = new HashMap<>(); + static { + acctTradeBotSuppliers.put(BitcoinACCTv1.class, BitcoinACCTv1TradeBot::getInstance); + acctTradeBotSuppliers.put(LitecoinACCTv1.class, LitecoinACCTv1TradeBot::getInstance); + acctTradeBotSuppliers.put(LitecoinACCTv2.class, LitecoinACCTv2TradeBot::getInstance); + acctTradeBotSuppliers.put(LitecoinACCTv3.class, LitecoinACCTv3TradeBot::getInstance); + acctTradeBotSuppliers.put(DogecoinACCTv1.class, DogecoinACCTv1TradeBot::getInstance); + acctTradeBotSuppliers.put(DogecoinACCTv2.class, DogecoinACCTv2TradeBot::getInstance); + acctTradeBotSuppliers.put(DogecoinACCTv3.class, DogecoinACCTv3TradeBot::getInstance); + } + + private static TradeBot instance; + + private final Map presenceTimestampsByAtAddress = Collections.synchronizedMap(new HashMap<>()); + + private TradeBot() { + EventBus.INSTANCE.addListener(event -> TradeBot.getInstance().listen(event)); + } + + public static synchronized TradeBot getInstance() { + if (instance == null) + instance = new TradeBot(); + + return instance; + } + + public ACCT getAcctUsingAtData(ATData atData) { + byte[] codeHash = atData.getCodeHash(); + if (codeHash == null) + return null; + + return SupportedBlockchain.getAcctByCodeHash(codeHash); + } + + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { + ACCT acct = this.getAcctUsingAtData(atData); + if (acct == null) + return null; + + return acct.populateTradeData(repository, atData); + } + + /** + * Creates a new trade-bot entry from the "Bob" viewpoint, + * i.e. OFFERing QORT in exchange for foreign blockchain currency. + *

+ * Generates: + *

    + *
  • new 'trade' private key
    • + *
  • secret(s)
    • + *
+ * Derives: + *
    + *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • + *
  • 'foreign' public key, public key hash
    • + *
  • hash(es) of secret(s)
    • + *
+ * A Qortal AT is then constructed including the following as constants in the 'data segment': + *
    + *
  • 'native' (Qortal) 'trade' address - used to MESSAGE AT
    • + *
  • 'foreign' public key hash - used by Alice's to allow redeem of currency on foreign blockchain
    • + *
  • hash(es) of secret(s) - used by AT (optional) and foreign blockchain as needed
    • + *
  • QORT amount on offer by Bob
    • + *
  • foreign currency amount expected in return by Bob (from Alice)
    • + *
  • trading timeout, in case things go wrong and everyone needs to refund
    • + *
+ * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. + *

+ * Trade-bot will wait for Bob's AT to be deployed before taking next step. + *

+ * @param repository + * @param tradeBotCreateRequest + * @return raw, unsigned DEPLOY_AT transaction + * @throws DataException + */ + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { + // Fetch latest ACCT version for requested foreign blockchain + ACCT acct = tradeBotCreateRequest.foreignBlockchain.getLatestAcct(); + + AcctTradeBot acctTradeBot = findTradeBotForAcct(acct); + if (acctTradeBot == null) + return null; + + return acctTradeBot.createTrade(repository, tradeBotCreateRequest); + } + + /** + * Creates a trade-bot entry from the 'Alice' viewpoint, + * i.e. matching foreign blockchain currency to an existing QORT offer. + *

+ * Requires a chosen trade offer from Bob, passed by crossChainTradeData + * and access to a foreign blockchain wallet via foreignKey. + *

+ * @param repository + * @param crossChainTradeData chosen trade OFFER that Alice wants to match + * @param foreignKey foreign blockchain wallet key + * @throws DataException + */ + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, + CrossChainTradeData crossChainTradeData, String foreignKey, String receivingAddress) throws DataException { + AcctTradeBot acctTradeBot = findTradeBotForAcct(acct); + if (acctTradeBot == null) { + LOGGER.debug(() -> String.format("Couldn't find ACCT trade-bot for AT %s", atData.getATAddress())); + return ResponseResult.NETWORK_ISSUE; + } + + // Check Alice doesn't already have an existing, on-going trade-bot entry for this AT. + if (repository.getCrossChainRepository().existsTradeWithAtExcludingStates(atData.getATAddress(), acctTradeBot.getEndStates())) + return ResponseResult.TRADE_ALREADY_EXISTS; + + return acctTradeBot.startResponse(repository, atData, acct, crossChainTradeData, foreignKey, receivingAddress); + } + + public boolean deleteEntry(Repository repository, byte[] tradePrivateKey) throws DataException { + TradeBotData tradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + if (tradeBotData == null) + // Can't delete what we don't have! + return false; + + boolean canDelete = false; + + ACCT acct = SupportedBlockchain.getAcctByName(tradeBotData.getAcctName()); + if (acct == null) + // We can't/no longer support this ACCT + canDelete = true; + else { + AcctTradeBot acctTradeBot = findTradeBotForAcct(acct); + canDelete = acctTradeBot == null || acctTradeBot.canDelete(repository, tradeBotData); + } + + if (canDelete) { + repository.getCrossChainRepository().delete(tradePrivateKey); + repository.saveChanges(); + } + + return canDelete; + } + + @Override + public void listen(Event event) { + if (!(event instanceof Controller.NewBlockEvent)) + return; + + synchronized (this) { + List allTradeBotData; + + try (final Repository repository = RepositoryManager.getRepository()) { + allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + } catch (DataException e) { + LOGGER.error("Couldn't run trade bot due to repository issue", e); + return; + } + + for (TradeBotData tradeBotData : allTradeBotData) + try (final Repository repository = RepositoryManager.getRepository()) { + // Find ACCT-specific trade-bot for this entry + ACCT acct = SupportedBlockchain.getAcctByName(tradeBotData.getAcctName()); + if (acct == null) { + LOGGER.debug(() -> String.format("Couldn't find ACCT matching name %s", tradeBotData.getAcctName())); + continue; + } + + AcctTradeBot acctTradeBot = findTradeBotForAcct(acct); + if (acctTradeBot == null) { + LOGGER.debug(() -> String.format("Couldn't find ACCT trade-bot matching name %s", tradeBotData.getAcctName())); + continue; + } + + acctTradeBot.progress(repository, tradeBotData); + } catch (DataException e) { + LOGGER.error("Couldn't run trade bot due to repository issue", e); + } catch (ForeignBlockchainException e) { + LOGGER.warn(() -> String.format("Foreign blockchain issue processing trade-bot entry for AT %s: %s", tradeBotData.getAtAddress(), e.getMessage())); + } + } + } + + public static byte[] generateTradePrivateKey() { + // The private key is used for both Curve25519 and secp256k1 so needs to be valid for both. + // Curve25519 accepts any seed, so generate a valid secp256k1 key and use that. + return new ECKey().getPrivKeyBytes(); + } + + public static byte[] deriveTradeNativePublicKey(byte[] privateKey) { + return PrivateKeyAccount.toPublicKey(privateKey); + } + + public static byte[] deriveTradeForeignPublicKey(byte[] privateKey) { + return ECKey.fromPrivate(privateKey).getPubKey(); + } + + /*package*/ static byte[] generateSecret() { + byte[] secret = new byte[32]; + RANDOM.nextBytes(secret); + return secret; + } + + /*package*/ static void backupTradeBotData(Repository repository, List additional) { + // Attempt to backup the trade bot data. This an optional step and doesn't impact trading, so don't throw an exception on failure + try { + LOGGER.info("About to backup trade bot data..."); + HSQLDBImportExport.backupTradeBotStates(repository, additional); + } catch (DataException e) { + LOGGER.info(String.format("Repository issue when exporting trade bot data: %s", e.getMessage())); + } + } + + /** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */ + /*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, + String newState, int newStateValue, Supplier logMessageSupplier) throws DataException { + tradeBotData.setState(newState); + tradeBotData.setStateValue(newStateValue); + tradeBotData.setTimestamp(NTP.getTime()); + repository.getCrossChainRepository().save(tradeBotData); + repository.saveChanges(); + + if (Settings.getInstance().isTradebotSystrayEnabled()) + SysTray.getInstance().showMessage("Trade-Bot", String.format("%s: %s", tradeBotData.getAtAddress(), newState), MessageType.INFO); + + if (logMessageSupplier != null) + LOGGER.info(logMessageSupplier); + + LOGGER.debug(() -> String.format("new state for trade-bot entry based on AT %s: %s", tradeBotData.getAtAddress(), newState)); + + notifyStateChange(tradeBotData); + } + + /** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */ + /*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, StateNameAndValueSupplier newStateSupplier, Supplier logMessageSupplier) throws DataException { + updateTradeBotState(repository, tradeBotData, newStateSupplier.getState(), newStateSupplier.getStateValue(), logMessageSupplier); + } + + /** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */ + /*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, Supplier logMessageSupplier) throws DataException { + updateTradeBotState(repository, tradeBotData, tradeBotData.getState(), tradeBotData.getStateValue(), logMessageSupplier); + } + + /*package*/ static void notifyStateChange(TradeBotData tradeBotData) { + StateChangeEvent stateChangeEvent = new StateChangeEvent(tradeBotData); + EventBus.INSTANCE.notify(stateChangeEvent); + } + + /*package*/ static AcctTradeBot findTradeBotForAcct(ACCT acct) { + Supplier acctTradeBotSupplier = acctTradeBotSuppliers.get(acct.getClass()); + if (acctTradeBotSupplier == null) + return null; + + return acctTradeBotSupplier.get(); + } + + // PRESENCE-related + /*package*/ void updatePresence(Repository repository, TradeBotData tradeBotData, CrossChainTradeData tradeData) + throws DataException { + String atAddress = tradeBotData.getAtAddress(); + + PrivateKeyAccount tradeNativeAccount = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + String signerAddress = tradeNativeAccount.getAddress(); + + /* + * There's no point in Alice trying to build a PRESENCE transaction + * for an AT that isn't locked to her, as other peers won't be able + * to validate the PRESENCE transaction as signing public key won't + * be visible. + */ + if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) + // Signer is neither Bob, nor Alice, or trade not yet locked to Alice + return; + + long now = NTP.getTime(); + long threshold = now - PresenceType.TRADE_BOT.getLifetime(); + + long timestamp = presenceTimestampsByAtAddress.compute(atAddress, (k, v) -> (v == null || v < threshold) ? now : v); + + // If timestamp hasn't been updated then nothing to do + if (timestamp != now) + return; + + int txGroupId = Group.NO_GROUP; + byte[] reference = new byte[TransactionTransformer.SIGNATURE_LENGTH]; + byte[] creatorPublicKey = tradeNativeAccount.getPublicKey(); + long fee = 0L; + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, null); + + int nonce = 0; + byte[] timestampSignature = tradeNativeAccount.sign(Longs.toByteArray(timestamp)); + + PresenceTransactionData transactionData = new PresenceTransactionData(baseTransactionData, nonce, PresenceType.TRADE_BOT, timestampSignature); + + PresenceTransaction presenceTransaction = new PresenceTransaction(repository, transactionData); + presenceTransaction.computeNonce(); + + presenceTransaction.sign(tradeNativeAccount); + + ValidationResult result = presenceTransaction.importAsUnconfirmed(); + if (result != ValidationResult.OK) + LOGGER.debug(() -> String.format("Unable to build trade-bot PRESENCE transaction for %s: %s", tradeBotData.getAtAddress(), result.name())); + } + +} diff --git a/src/main/java/org/qortal/crosschain/ACCT.java b/src/main/java/org/qortal/crosschain/ACCT.java new file mode 100644 index 000000000..de28cfce1 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/ACCT.java @@ -0,0 +1,25 @@ +package org.qortal.crosschain; + +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; + +public interface ACCT { + + public byte[] getCodeBytesHash(); + + public int getModeByteOffset(); + + public ForeignBlockchain getBlockchain(); + + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException; + + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException; + + public byte[] buildCancelMessage(String creatorQortalAddress); + + public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException; + +} diff --git a/src/main/java/org/qortal/crosschain/AcctMode.java b/src/main/java/org/qortal/crosschain/AcctMode.java new file mode 100644 index 000000000..214960326 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/AcctMode.java @@ -0,0 +1,21 @@ +package org.qortal.crosschain; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +import java.util.Map; + +public enum AcctMode { + OFFERING(0), TRADING(1), CANCELLED(2), REFUNDED(3), REDEEMED(4); + + public final int value; + private static final Map map = stream(AcctMode.values()).collect(toMap(mode -> mode.value, mode -> mode)); + + AcctMode(int value) { + this.value = value; + } + + public static AcctMode valueOf(int value) { + return map.get(value); + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/BTC.java b/src/main/java/org/qortal/crosschain/BTC.java deleted file mode 100644 index ec53eb08c..000000000 --- a/src/main/java/org/qortal/crosschain/BTC.java +++ /dev/null @@ -1,159 +0,0 @@ -package org.qortal.crosschain; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.core.Transaction; -import org.bitcoinj.core.TransactionOutput; -import org.bitcoinj.params.MainNetParams; -import org.bitcoinj.params.RegTestParams; -import org.bitcoinj.params.TestNet3Params; -import org.bitcoinj.script.ScriptBuilder; -import org.bitcoinj.utils.MonetaryFormat; -import org.qortal.settings.Settings; -import org.qortal.utils.BitTwiddling; -import org.qortal.utils.Pair; - -public class BTC { - - public static final MonetaryFormat FORMAT = new MonetaryFormat().minDecimals(8).postfixCode(); - public static final long NO_LOCKTIME_NO_RBF_SEQUENCE = 0xFFFFFFFFL; - public static final long LOCKTIME_NO_RBF_SEQUENCE = NO_LOCKTIME_NO_RBF_SEQUENCE - 1; - public static final int HASH160_LENGTH = 20; - - protected static final Logger LOGGER = LogManager.getLogger(BTC.class); - - private static final int TIMESTAMP_OFFSET = 4 + 32 + 32; - - public enum BitcoinNet { - MAIN { - @Override - public NetworkParameters getParams() { - return MainNetParams.get(); - } - }, - TEST3 { - @Override - public NetworkParameters getParams() { - return TestNet3Params.get(); - } - }, - REGTEST { - @Override - public NetworkParameters getParams() { - return RegTestParams.get(); - } - }; - - public abstract NetworkParameters getParams(); - } - - private static BTC instance; - private final NetworkParameters params; - private final ElectrumX electrumX; - - // Constructors and instance - - private BTC() { - BitcoinNet bitcoinNet = Settings.getInstance().getBitcoinNet(); - this.params = bitcoinNet.getParams(); - - LOGGER.info(() -> String.format("Starting Bitcoin support using %s", bitcoinNet.name())); - - this.electrumX = ElectrumX.getInstance(bitcoinNet.name()); - } - - public static synchronized BTC getInstance() { - if (instance == null) - instance = new BTC(); - - return instance; - } - - // Getters & setters - - public NetworkParameters getNetworkParameters() { - return this.params; - } - - public static synchronized void resetForTesting() { - instance = null; - } - - // Actual useful methods for use by other classes - - /** Returns median timestamp from latest 11 blocks, in seconds. */ - public Integer getMedianBlockTime() { - Integer height = this.electrumX.getCurrentHeight(); - if (height == null) - return null; - - // Grab latest 11 blocks - List blockHeaders = this.electrumX.getBlockHeaders(height - 11, 11); - if (blockHeaders == null || blockHeaders.size() < 11) - return null; - - List blockTimestamps = blockHeaders.stream().map(blockHeader -> BitTwiddling.fromLEBytes(blockHeader, TIMESTAMP_OFFSET)).collect(Collectors.toList()); - - // Descending, but order shouldn't matter as we're picking median... - blockTimestamps.sort((a, b) -> Integer.compare(b, a)); - - return blockTimestamps.get(5); - } - - public Coin getBalance(String base58Address) { - Long balance = this.electrumX.getBalance(addressToScript(base58Address)); - if (balance == null) - return null; - - return Coin.valueOf(balance); - } - - public List getUnspentOutputs(String base58Address) { - List> unspentOutputs = this.electrumX.getUnspentOutputs(addressToScript(base58Address)); - if (unspentOutputs == null) - return null; - - List unspentTransactionOutputs = new ArrayList<>(); - for (Pair unspentOutput : unspentOutputs) { - List transactionOutputs = getOutputs(unspentOutput.getA()); - if (transactionOutputs == null) - return null; - - unspentTransactionOutputs.add(transactionOutputs.get(unspentOutput.getB())); - } - - return unspentTransactionOutputs; - } - - public List getOutputs(byte[] txHash) { - byte[] rawTransactionBytes = this.electrumX.getRawTransaction(txHash); - if (rawTransactionBytes == null) - return null; - - Transaction transaction = new Transaction(this.params, rawTransactionBytes); - return transaction.getOutputs(); - } - - public List getAddressTransactions(String base58Address) { - return this.electrumX.getAddressTransactions(addressToScript(base58Address)); - } - - public boolean broadcastTransaction(Transaction transaction) { - return this.electrumX.broadcastTransaction(transaction.bitcoinSerialize()); - } - - // Utility methods for us - - private byte[] addressToScript(String base58Address) { - Address address = Address.fromString(this.params, base58Address); - return ScriptBuilder.createOutputScript(address).getProgram(); - } - -} diff --git a/src/main/java/org/qortal/crosschain/BTCACCT.java b/src/main/java/org/qortal/crosschain/BTCACCT.java deleted file mode 100644 index a0246d04e..000000000 --- a/src/main/java/org/qortal/crosschain/BTCACCT.java +++ /dev/null @@ -1,669 +0,0 @@ -package org.qortal.crosschain; - -import static org.ciyam.at.OpCode.calcOffset; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; -import java.util.function.Function; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.ECKey; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.core.Transaction; -import org.bitcoinj.core.Transaction.SigHash; -import org.bitcoinj.core.TransactionInput; -import org.bitcoinj.core.TransactionOutput; -import org.bitcoinj.crypto.TransactionSignature; -import org.bitcoinj.script.Script; -import org.bitcoinj.script.ScriptBuilder; -import org.bitcoinj.script.ScriptChunk; -import org.bitcoinj.script.ScriptOpCodes; -import org.ciyam.at.API; -import org.ciyam.at.CompilationException; -import org.ciyam.at.FunctionCode; -import org.ciyam.at.MachineState; -import org.ciyam.at.OpCode; -import org.ciyam.at.Timestamp; -import org.qortal.account.Account; -import org.qortal.asset.Asset; -import org.qortal.at.QortalAtLoggerFactory; -import org.qortal.block.BlockChain; -import org.qortal.block.BlockChain.CiyamAtSettings; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.at.ATStateData; -import org.qortal.data.block.BlockData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.utils.Base58; -import org.qortal.utils.BitTwiddling; - -import com.google.common.hash.HashCode; -import com.google.common.primitives.Bytes; - -/* - * Bob generates Bitcoin private key - * private key required to sign P2SH redeem tx - * private key can be used to create 'secret' (e.g. double-SHA256) - * encrypted private key could be stored in Qortal AT for access by Bob from any node - * Bob creates Qortal AT - * Alice finds Qortal AT and wants to trade - * Alice generates Bitcoin private key - * Alice will need to send Bob her Qortal address and Bitcoin refund address - * Bob sends Alice's Qortal address to Qortal AT - * Qortal AT sends initial QORT payment to Alice (so she has QORT to send message to AT and claim funds) - * Alice receives funds and checks Qortal AT to confirm it's locked to her - * Alice creates/funds Bitcoin P2SH - * Alice requires: Bob's redeem Bitcoin address, Alice's refund Bitcoin address, derived locktime - * Bob checks P2SH is funded - * Bob requires: Bob's redeem Bitcoin address, Alice's refund Bitcoin address, derived locktime - * Bob uses secret to redeem P2SH - * Qortal core/UI will need to create, and sign, this transaction - * Alice scans P2SH redeem tx and uses secret to redeem Qortal AT - */ - -public class BTCACCT { - - public static final int SECRET_LENGTH = 32; - public static final int MIN_LOCKTIME = 1500000000; - public static final byte[] CODE_BYTES_HASH = HashCode.fromString("edcdb1feb36e079c5f956faff2f24219b12e5fbaaa05654335e615e33218282f").asBytes(); // SHA256 of AT code bytes - - /* - * OP_TUCK (to copy public key to before signature) - * OP_CHECKSIGVERIFY (sig & pubkey must verify or script fails) - * OP_HASH160 (convert public key to PKH) - * OP_DUP (duplicate PKH) - * OP_EQUAL (does PKH match refund PKH?) - * OP_IF - * OP_DROP (no need for duplicate PKH) - * - * OP_CHECKLOCKTIMEVERIFY (if this passes, leftover stack is so script passes) - * OP_ELSE - * OP_EQUALVERIFY (duplicate PKH must match redeem PKH or script fails) - * OP_HASH160 (hash secret) - * OP_EQUAL (do hashes of secrets match? if true, script passes else script fails) - * OP_ENDIF - */ - - private static final byte[] redeemScript1 = HashCode.fromString("7dada97614").asBytes(); // OP_TUCK OP_CHECKSIGVERIFY OP_HASH160 OP_DUP push(0x14 bytes) - private static final byte[] redeemScript2 = HashCode.fromString("87637504").asBytes(); // OP_EQUAL OP_IF OP_DROP push(0x4 bytes) - private static final byte[] redeemScript3 = HashCode.fromString("b16714").asBytes(); // OP_CHECKLOCKTIMEVERIFY OP_ELSE push(0x14 bytes) - private static final byte[] redeemScript4 = HashCode.fromString("88a914").asBytes(); // OP_EQUALVERIFY OP_HASH160 push(0x14 bytes) - private static final byte[] redeemScript5 = HashCode.fromString("8768").asBytes(); // OP_EQUAL OP_ENDIF - - /** - * Returns Bitcoin redeemScript used for cross-chain trading. - *

- * See comments in {@link BTCACCT} for more details. - * - * @param refunderPubKeyHash 20-byte HASH160 of P2SH funder's public key, for refunding purposes - * @param lockTime seconds-since-epoch threshold, after which P2SH funder can claim refund - * @param redeemerPubKeyHash 20-byte HASH160 of P2SH redeemer's public key - * @param secretHash 20-byte HASH160 of secret, used by P2SH redeemer to claim funds - * @return - */ - public static byte[] buildScript(byte[] refunderPubKeyHash, int lockTime, byte[] redeemerPubKeyHash, byte[] secretHash) { - return Bytes.concat(redeemScript1, refunderPubKeyHash, redeemScript2, BitTwiddling.toLEByteArray((int) (lockTime & 0xffffffffL)), - redeemScript3, redeemerPubKeyHash, redeemScript4, secretHash, redeemScript5); - } - - /** - * Builds a custom transaction to spend P2SH. - * - * @param amount output amount, should be total of input amounts, less miner fees - * @param spendKey key for signing transaction, and also where funds are 'sent' (output) - * @param fundingOutput output from transaction that funded P2SH address - * @param redeemScriptBytes the redeemScript itself, in byte[] form - * @param lockTime (optional) transaction nLockTime, used in refund scenario - * @param scriptSigBuilder function for building scriptSig using transaction input signature - * @return Signed Bitcoin transaction for spending P2SH - */ - public static Transaction buildP2shTransaction(Coin amount, ECKey spendKey, List fundingOutputs, byte[] redeemScriptBytes, Long lockTime, Function scriptSigBuilder) { - NetworkParameters params = BTC.getInstance().getNetworkParameters(); - - Transaction transaction = new Transaction(params); - transaction.setVersion(2); - - // Output is back to P2SH funder - transaction.addOutput(amount, ScriptBuilder.createP2PKHOutputScript(spendKey.getPubKeyHash())); - - for (int inputIndex = 0; inputIndex < fundingOutputs.size(); ++inputIndex) { - TransactionOutput fundingOutput = fundingOutputs.get(inputIndex); - - // Input (without scriptSig prior to signing) - TransactionInput input = new TransactionInput(params, null, redeemScriptBytes, fundingOutput.getOutPointFor()); - if (lockTime != null) - input.setSequenceNumber(BTC.LOCKTIME_NO_RBF_SEQUENCE); // Use max-value, so no lockTime and no RBF - else - input.setSequenceNumber(BTC.NO_LOCKTIME_NO_RBF_SEQUENCE); // Use max-value - 1, so lockTime can be used but not RBF - transaction.addInput(input); - } - - // Set locktime after inputs added but before input signatures are generated - if (lockTime != null) - transaction.setLockTime(lockTime); - - for (int inputIndex = 0; inputIndex < fundingOutputs.size(); ++inputIndex) { - // Generate transaction signature for input - final boolean anyoneCanPay = false; - TransactionSignature txSig = transaction.calculateSignature(inputIndex, spendKey, redeemScriptBytes, SigHash.ALL, anyoneCanPay); - - // Calculate transaction signature - byte[] txSigBytes = txSig.encodeToBitcoin(); - - // Build scriptSig using lambda and tx signature - Script scriptSig = scriptSigBuilder.apply(txSigBytes); - - // Set input scriptSig - transaction.getInput(inputIndex).setScriptSig(scriptSig); - } - - return transaction; - } - - /** - * Returns signed Bitcoin transaction claiming refund from P2SH address. - * - * @param refundAmount refund amount, should be total of input amounts, less miner fees - * @param refundKey key for signing transaction, and also where refund is 'sent' (output) - * @param fundingOutput output from transaction that funded P2SH address - * @param redeemScriptBytes the redeemScript itself, in byte[] form - * @param lockTime transaction nLockTime - must be at least locktime used in redeemScript - * @return Signed Bitcoin transaction for refunding P2SH - */ - public static Transaction buildRefundTransaction(Coin refundAmount, ECKey refundKey, List fundingOutputs, byte[] redeemScriptBytes, long lockTime) { - Function refundSigScriptBuilder = (txSigBytes) -> { - // Build scriptSig with... - ScriptBuilder scriptBuilder = new ScriptBuilder(); - - // transaction signature - scriptBuilder.addChunk(new ScriptChunk(txSigBytes.length, txSigBytes)); - - // redeem public key - byte[] refundPubKey = refundKey.getPubKey(); - scriptBuilder.addChunk(new ScriptChunk(refundPubKey.length, refundPubKey)); - - // redeem script - scriptBuilder.addChunk(new ScriptChunk(ScriptOpCodes.OP_PUSHDATA1, redeemScriptBytes)); - - return scriptBuilder.build(); - }; - - return buildP2shTransaction(refundAmount, refundKey, fundingOutputs, redeemScriptBytes, lockTime, refundSigScriptBuilder); - } - - /** - * Returns signed Bitcoin transaction redeeming funds from P2SH address. - * - * @param redeemAmount redeem amount, should be total of input amounts, less miner fees - * @param redeemKey key for signing transaction, and also where funds are 'sent' (output) - * @param fundingOutput output from transaction that funded P2SH address - * @param redeemScriptBytes the redeemScript itself, in byte[] form - * @param secret actual 32-byte secret used when building redeemScript - * @return Signed Bitcoin transaction for redeeming P2SH - */ - public static Transaction buildRedeemTransaction(Coin redeemAmount, ECKey redeemKey, List fundingOutputs, byte[] redeemScriptBytes, byte[] secret) { - Function redeemSigScriptBuilder = (txSigBytes) -> { - // Build scriptSig with... - ScriptBuilder scriptBuilder = new ScriptBuilder(); - - // secret - scriptBuilder.addChunk(new ScriptChunk(secret.length, secret)); - - // transaction signature - scriptBuilder.addChunk(new ScriptChunk(txSigBytes.length, txSigBytes)); - - // redeem public key - byte[] redeemPubKey = redeemKey.getPubKey(); - scriptBuilder.addChunk(new ScriptChunk(redeemPubKey.length, redeemPubKey)); - - // redeem script - scriptBuilder.addChunk(new ScriptChunk(ScriptOpCodes.OP_PUSHDATA1, redeemScriptBytes)); - - return scriptBuilder.build(); - }; - - return buildP2shTransaction(redeemAmount, redeemKey, fundingOutputs, redeemScriptBytes, null, redeemSigScriptBuilder); - } - - /** - * Returns Qortal AT creation bytes for cross-chain trading AT. - *

- * tradeTimeout (minutes) is the time window for the recipient to send the - * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. - * - * @param qortalCreator Qortal address for AT creator, also used for refunds - * @param secretHash 20-byte HASH160 of 32-byte secret - * @param tradeTimeout how many minutes, from start of 'trade mode' until AT auto-refunds AT creator - * @param initialPayout how much QORT to pay trade partner upon switch to 'trade mode' - * @param redeemPayout how much QORT to pay trade partner if they send correct 32-byte secret to AT - * @param bitcoinAmount how much BTC the AT creator is expecting to trade - * @return - */ - public static byte[] buildQortalAT(String qortalCreator, byte[] secretHash, int tradeTimeout, long initialPayout, long redeemPayout, long bitcoinAmount) { - // Labels for data segment addresses - int addrCounter = 0; - - // Constants (with corresponding dataByteBuffer.put*() calls below) - - final int addrQortalCreator1 = addrCounter++; - final int addrQortalCreator2 = addrCounter++; - final int addrQortalCreator3 = addrCounter++; - final int addrQortalCreator4 = addrCounter++; - - final int addrSecretHash = addrCounter; - addrCounter += 4; - - final int addrTradeTimeout = addrCounter++; - final int addrInitialPayoutAmount = addrCounter++; - final int addrRedeemPayoutAmount = addrCounter++; - final int addrBitcoinAmount = addrCounter++; - - final int addrMessageTxType = addrCounter++; - - final int addrSecretHashPointer = addrCounter++; - final int addrQortalRecipientPointer = addrCounter++; - final int addrMessageSenderPointer = addrCounter++; - - final int addrMessageDataPointer = addrCounter++; - final int addrMessageDataLength = addrCounter++; - - final int addrEndOfConstants = addrCounter; - - // Variables - - final int addrQortalRecipient1 = addrCounter++; - final int addrQortalRecipient2 = addrCounter++; - final int addrQortalRecipient3 = addrCounter++; - final int addrQortalRecipient4 = addrCounter++; - - final int addrTradeRefundTimestamp = addrCounter++; - final int addrLastTxTimestamp = addrCounter++; - final int addrBlockTimestamp = addrCounter++; - final int addrTxType = addrCounter++; - final int addrResult = addrCounter++; - - final int addrMessageSender1 = addrCounter++; - final int addrMessageSender2 = addrCounter++; - final int addrMessageSender3 = addrCounter++; - final int addrMessageSender4 = addrCounter++; - - final int addrMessageData = addrCounter; - addrCounter += 4; - - // Data segment - ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); - - // AT creator's Qortal address, decoded from Base58 - assert dataByteBuffer.position() == addrQortalCreator1 * MachineState.VALUE_SIZE : "addrQortalCreator1 incorrect"; - byte[] qortalCreatorBytes = Base58.decode(qortalCreator); - dataByteBuffer.put(Bytes.ensureCapacity(qortalCreatorBytes, 32, 0)); - - // Hash of secret - assert dataByteBuffer.position() == addrSecretHash * MachineState.VALUE_SIZE : "addrSecretHash incorrect"; - dataByteBuffer.put(Bytes.ensureCapacity(secretHash, 32, 0)); - - // Trade timeout in minutes - assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; - dataByteBuffer.putLong(tradeTimeout); - - // Initial payout amount - assert dataByteBuffer.position() == addrInitialPayoutAmount * MachineState.VALUE_SIZE : "addrInitialPayoutAmount incorrect"; - dataByteBuffer.putLong(initialPayout); - - // Redeem payout amount - assert dataByteBuffer.position() == addrRedeemPayoutAmount * MachineState.VALUE_SIZE : "addrRedeemPayoutAmount incorrect"; - dataByteBuffer.putLong(redeemPayout); - - // Expected Bitcoin amount - assert dataByteBuffer.position() == addrBitcoinAmount * MachineState.VALUE_SIZE : "addrBitcoinAmount incorrect"; - dataByteBuffer.putLong(bitcoinAmount); - - // We're only interested in MESSAGE transactions - assert dataByteBuffer.position() == addrMessageTxType * MachineState.VALUE_SIZE : "addrMessageTxType incorrect"; - dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); - - // Index into data segment of hash, used by GET_B_IND - assert dataByteBuffer.position() == addrSecretHashPointer * MachineState.VALUE_SIZE : "addrSecretHashPointer incorrect"; - dataByteBuffer.putLong(addrSecretHash); - - // Index into data segment of recipient address, used by SET_B_IND - assert dataByteBuffer.position() == addrQortalRecipientPointer * MachineState.VALUE_SIZE : "addrQortalRecipientPointer incorrect"; - dataByteBuffer.putLong(addrQortalRecipient1); - - // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND - assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; - dataByteBuffer.putLong(addrMessageSender1); - - // Source location and length for hashing any passed secret - assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; - dataByteBuffer.putLong(addrMessageData); - assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; - dataByteBuffer.putLong(32L); - - assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; - - // Code labels - Integer labelRefund = null; - - Integer labelOfferTxLoop = null; - Integer labelCheckOfferTx = null; - - Integer labelTradeMode = null; - Integer labelTradeTxLoop = null; - Integer labelCheckTradeTx = null; - - ByteBuffer codeByteBuffer = ByteBuffer.allocate(512); - - // Two-pass version - for (int pass = 0; pass < 2; ++pass) { - codeByteBuffer.clear(); - - try { - /* Initialization */ - - // Use AT creation 'timestamp' as starting point for finding transactions sent to AT - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxTimestamp)); - - // Set restart position to after this opcode - codeByteBuffer.put(OpCode.SET_PCS.compile()); - - /* Loop, waiting for message from AT owner containing trade partner details, or AT owner's address to cancel offer */ - - /* Transaction processing loop */ - labelOfferTxLoop = codeByteBuffer.position(); - - // Find next transaction to this AT since the last one (if any) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxTimestamp)); - // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); - // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction - codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckOfferTx))); - // Stop and wait for next block - codeByteBuffer.put(OpCode.STP_IMD.compile()); - - /* Check transaction */ - labelCheckOfferTx = codeByteBuffer.position(); - - // Update our 'last found transaction's timestamp' using 'timestamp' from transaction - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxTimestamp)); - // Extract transaction type (message/payment) from transaction and save type in addrTxType - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxType)); - // If transaction type is not MESSAGE type then go look for another transaction - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxType, addrMessageTxType, calcOffset(codeByteBuffer, labelOfferTxLoop))); - - /* Check transaction's sender */ - - // Extract sender address from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); - // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalCreator1, calcOffset(codeByteBuffer, labelOfferTxLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalCreator2, calcOffset(codeByteBuffer, labelOfferTxLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalCreator3, calcOffset(codeByteBuffer, labelOfferTxLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalCreator4, calcOffset(codeByteBuffer, labelOfferTxLoop))); - - /* Extract trade partner info from message */ - - // Extract message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrQortalRecipient1 (as pointed to by addrQortalRecipientPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalRecipientPointer)); - // Compare each of recipient address with creator's address (for offer-cancel scenario). If they don't match, assume recipient is trade partner. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrQortalRecipient1, addrQortalCreator1, calcOffset(codeByteBuffer, labelTradeMode))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrQortalRecipient2, addrQortalCreator2, calcOffset(codeByteBuffer, labelTradeMode))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrQortalRecipient3, addrQortalCreator3, calcOffset(codeByteBuffer, labelTradeMode))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrQortalRecipient4, addrQortalCreator4, calcOffset(codeByteBuffer, labelTradeMode))); - // Recipient address is AT creator's address, so cancel offer and finish. - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); - - /* Switch to 'trade mode' */ - labelTradeMode = codeByteBuffer.position(); - - // Send initial payment to recipient so they have enough funds to message AT if all goes well - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrInitialPayoutAmount)); - - // Calculate trade timeout refund 'timestamp' by adding addrTradeTimeout minutes to above message's 'timestamp', then save into addrTradeRefundTimestamp - codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrTradeRefundTimestamp, addrLastTxTimestamp, addrTradeTimeout)); - - // Set restart position to after this opcode - codeByteBuffer.put(OpCode.SET_PCS.compile()); - - /* Loop, waiting for trade timeout or message from Qortal trade recipient containing secret */ - - // Fetch current block 'timestamp' - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); - // If we're not past refund 'timestamp' then look for next transaction - codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrTradeRefundTimestamp, calcOffset(codeByteBuffer, labelTradeTxLoop))); - // We're past refund 'timestamp' so go refund everything back to AT creator - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); - - /* Transaction processing loop */ - labelTradeTxLoop = codeByteBuffer.position(); - - // Find next transaction to this AT since the last one (if any) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxTimestamp)); - // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); - // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction - codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTx))); - // Stop and wait for next block - codeByteBuffer.put(OpCode.STP_IMD.compile()); - - /* Check transaction */ - labelCheckTradeTx = codeByteBuffer.position(); - - // Update our 'last found transaction's timestamp' using 'timestamp' from transaction - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxTimestamp)); - // Extract transaction type (message/payment) from transaction and save type in addrTxType - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxType)); - // If transaction type is not MESSAGE type then go look for another transaction - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxType, addrMessageTxType, calcOffset(codeByteBuffer, labelTradeTxLoop))); - - /* Check transaction's sender */ - - // Extract sender address from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); - // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalRecipient1, calcOffset(codeByteBuffer, labelTradeTxLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalRecipient2, calcOffset(codeByteBuffer, labelTradeTxLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalRecipient3, calcOffset(codeByteBuffer, labelTradeTxLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalRecipient4, calcOffset(codeByteBuffer, labelTradeTxLoop))); - - /* Check 'secret' in transaction's message */ - - // Extract message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); - // Load B register with expected hash result (as pointed to by addrSecretHashPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrSecretHashPointer)); - // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). - // Save the equality result (1 if they match, 0 otherwise) into addrResult. - codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); - // If hashes don't match, addrResult will be zero so go find another transaction - codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelTradeTxLoop))); - - /* Success! Pay arranged amount to intended recipient */ - - // Load B register with intended recipient address (as pointed to by addrQortalRecipientPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrQortalRecipientPointer)); - // Pay AT's balance to recipient - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrRedeemPayoutAmount)); - // Fall-through to refunding any remaining balance back to AT creator - - /* Refund balance back to AT creator */ - labelRefund = codeByteBuffer.position(); - - // Load B register with AT creator's address. - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); - // Pay AT's balance back to AT's creator. - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PAY_ALL_TO_ADDRESS_IN_B)); - // We're finished forever - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - } catch (CompilationException e) { - throw new IllegalStateException("Unable to compile BTC-QORT ACCT?", e); - } - } - - codeByteBuffer.flip(); - - byte[] codeBytes = new byte[codeByteBuffer.limit()]; - codeByteBuffer.get(codeBytes); - - assert Arrays.equals(Crypto.digest(codeBytes), BTCACCT.CODE_BYTES_HASH) - : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); - - final short ciyamAtVersion = 2; - final short numCallStackPages = 0; - final short numUserStackPages = 0; - final long minActivationAmount = 0L; - - return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - * - * @param repository - * @param atAddress - * @throws DataException - */ - public static CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { - String atAddress = atData.getATAddress(); - - ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); - byte[] stateData = atStateData.getStateData(); - - QortalAtLoggerFactory loggerFactory = QortalAtLoggerFactory.getInstance(); - byte[] dataBytes = MachineState.extractDataBytes(loggerFactory, stateData); - - CrossChainTradeData tradeData = new CrossChainTradeData(); - tradeData.qortalAtAddress = atAddress; - tradeData.qortalCreator = Crypto.toAddress(atData.getCreatorPublicKey()); - tradeData.creationTimestamp = atData.getCreation(); - - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); - - ByteBuffer dataByteBuffer = ByteBuffer.wrap(dataBytes); - byte[] addressBytes = new byte[32]; - - // Skip AT creator address - dataByteBuffer.position(dataByteBuffer.position() + 32); - - // Hash of secret - tradeData.secretHash = new byte[20]; - dataByteBuffer.get(tradeData.secretHash); - dataByteBuffer.position(dataByteBuffer.position() + 32 - 20); // skip to 32 bytes - - // Trade timeout - tradeData.tradeRefundTimeout = dataByteBuffer.getLong(); - - // Initial payout - tradeData.initialPayout = dataByteBuffer.getLong(); - - // Redeem payout - tradeData.redeemPayout = dataByteBuffer.getLong(); - - // Expected BTC amount - tradeData.expectedBitcoin = dataByteBuffer.getLong(); - - // Skip MESSAGE transaction type - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to secretHash - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to Qortal recipient - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to message sender - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to message data - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip message data length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Qortal recipient (if any) - dataByteBuffer.get(addressBytes); - - // Trade offer timeout (AT 'timestamp' converted to Qortal block height) - long tradeRefundTimestamp = dataByteBuffer.getLong(); - - if (tradeRefundTimestamp != 0) { - tradeData.mode = CrossChainTradeData.Mode.TRADE; - tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; - - if (addressBytes[0] != 0) - tradeData.qortalRecipient = Base58.encode(Arrays.copyOf(addressBytes, Account.ADDRESS_LENGTH)); - - // We'll suggest half of trade timeout - CiyamAtSettings ciyamAtSettings = BlockChain.getInstance().getCiyamAtSettings(); - - int tradeModeSwitchHeight = (int) (tradeData.tradeRefundHeight - tradeData.tradeRefundTimeout / ciyamAtSettings.minutesPerBlock); - - BlockData blockData = repository.getBlockRepository().fromHeight(tradeModeSwitchHeight); - if (blockData != null) { - tradeData.tradeModeTimestamp = blockData.getTimestamp(); // NOTE: milliseconds from epoch - tradeData.lockTime = (int) (tradeData.tradeModeTimestamp / 1000L + tradeData.tradeRefundTimeout / 2 * 60); - } - } else { - tradeData.mode = CrossChainTradeData.Mode.OFFER; - } - - return tradeData; - } - - public static byte[] findP2shSecret(String p2shAddress, List rawTransactions) { - NetworkParameters params = BTC.getInstance().getNetworkParameters(); - - for (byte[] rawTransaction : rawTransactions) { - Transaction transaction = new Transaction(params, rawTransaction); - - // Cycle through inputs, looking for one that spends our P2SH - for (TransactionInput input : transaction.getInputs()) { - Script scriptSig = input.getScriptSig(); - List scriptChunks = scriptSig.getChunks(); - - // Expected number of script chunks for redeem. Refund might not have the same number. - int expectedChunkCount = 1 /*secret*/ + 1 /*sig*/ + 1 /*pubkey*/ + 1 /*redeemScript*/; - if (scriptChunks.size() != expectedChunkCount) - continue; - - // We're expecting last chunk to contain the actual redeemScript - ScriptChunk lastChunk = scriptChunks.get(scriptChunks.size() - 1); - byte[] redeemScriptBytes = lastChunk.data; - - // If non-push scripts, redeemScript will be null - if (redeemScriptBytes == null) - continue; - - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - Address inputAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - - if (!inputAddress.toString().equals(p2shAddress)) - // Input isn't spending our P2SH - continue; - - byte[] secret = scriptChunks.get(0).data; - if (secret.length != BTCACCT.SECRET_LENGTH) - continue; - - return secret; - } - } - - return null; - } - -} diff --git a/src/main/java/org/qortal/crosschain/Bitcoin.java b/src/main/java/org/qortal/crosschain/Bitcoin.java new file mode 100644 index 000000000..6d904eb40 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/Bitcoin.java @@ -0,0 +1,188 @@ +package org.qortal.crosschain; + +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumMap; +import java.util.Map; + +import org.bitcoinj.core.Context; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.params.MainNetParams; +import org.bitcoinj.params.RegTestParams; +import org.bitcoinj.params.TestNet3Params; +import org.qortal.crosschain.ElectrumX.Server; +import org.qortal.crosschain.ElectrumX.Server.ConnectionType; +import org.qortal.settings.Settings; + +public class Bitcoin extends Bitcoiny { + + public static final String CURRENCY_CODE = "BTC"; + + // Temporary values until a dynamic fee system is written. + private static final long OLD_FEE_AMOUNT = 4_000L; // Not 5000 so that existing P2SH-B can output 1000, avoiding dust issue, leaving 4000 for fees. + private static final long NEW_FEE_TIMESTAMP = 1598280000000L; // milliseconds since epoch + private static final long NEW_FEE_AMOUNT = 10_000L; + + private static final long NON_MAINNET_FEE = 1000L; // enough for TESTNET3 and should be OK for REGTEST + + private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ElectrumX.Server.ConnectionType.class); + static { + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.TCP, 50001); + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.SSL, 50002); + } + + public enum BitcoinNet { + MAIN { + @Override + public NetworkParameters getParams() { + return MainNetParams.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + // Servers chosen on NO BASIS WHATSOEVER from various sources! + new Server("hodlers.beer", Server.ConnectionType.SSL, 50002), + new Server("btc.lastingcoin.net", Server.ConnectionType.SSL, 50002), + new Server("electrum.bitaroo.net", Server.ConnectionType.SSL, 50002), + new Server("bitcoin.grey.pw", Server.ConnectionType.SSL, 50002), + new Server("185.64.116.15", Server.ConnectionType.SSL, 50002), + new Server("alviss.coinjoined.com", Server.ConnectionType.SSL, 50002), + new Server("btc.litepay.ch", Server.ConnectionType.SSL, 50002), + new Server("xtrum.com", Server.ConnectionType.SSL, 50002), + new Server("electrum.acinq.co", Server.ConnectionType.SSL, 50002), + new Server("caleb.vegas", Server.ConnectionType.SSL, 50002), + new Server("electrum.coinext.com.br", Server.ConnectionType.TCP, 50001), + new Server("korea.electrum-server.com", Server.ConnectionType.TCP, 50001), + new Server("eai.coincited.net", Server.ConnectionType.TCP, 50001), + new Server("electrum.coinext.com.br", Server.ConnectionType.SSL, 50002), + new Server("node1.btccuracao.com", Server.ConnectionType.SSL, 50002), + new Server("korea.electrum-server.com", Server.ConnectionType.SSL, 50002), + new Server("btce.iiiiiii.biz", Server.ConnectionType.SSL, 50002), + new Server("bitcoin.lukechilds.co", Server.ConnectionType.SSL, 50002), + new Server("guichet.centure.cc", Server.ConnectionType.SSL, 50002), + new Server("electrumx.hodlwallet.com", Server.ConnectionType.SSL, 50002), + new Server("eai.coincited.net", Server.ConnectionType.SSL, 50002), + new Server("prospero.bitsrc.net", Server.ConnectionType.SSL, 50002), + new Server("gd42.org", Server.ConnectionType.SSL, 50002), + new Server("electrum.pabu.io", Server.ConnectionType.SSL, 50002)); + } + + @Override + public String getGenesisHash() { + return "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"; + } + + @Override + public long getP2shFee(Long timestamp) { + // TODO: This will need to be replaced with something better in the near future! + if (timestamp != null && timestamp < NEW_FEE_TIMESTAMP) + return OLD_FEE_AMOUNT; + + return NEW_FEE_AMOUNT; + } + }, + TEST3 { + @Override + public NetworkParameters getParams() { + return TestNet3Params.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + new Server("tn.not.fyi", Server.ConnectionType.SSL, 55002), + new Server("electrumx-test.1209k.com", Server.ConnectionType.SSL, 50002), + new Server("testnet.qtornado.com", Server.ConnectionType.SSL, 51002), + new Server("testnet.aranguren.org", Server.ConnectionType.TCP, 51001), + new Server("testnet.aranguren.org", Server.ConnectionType.SSL, 51002), + new Server("testnet.hsmiths.com", Server.ConnectionType.SSL, 53012)); + } + + @Override + public String getGenesisHash() { + return "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943"; + } + + @Override + public long getP2shFee(Long timestamp) { + return NON_MAINNET_FEE; + } + }, + REGTEST { + @Override + public NetworkParameters getParams() { + return RegTestParams.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + new Server("localhost", Server.ConnectionType.TCP, 50001), + new Server("localhost", Server.ConnectionType.SSL, 50002)); + } + + @Override + public String getGenesisHash() { + // This is unique to each regtest instance + return null; + } + + @Override + public long getP2shFee(Long timestamp) { + return NON_MAINNET_FEE; + } + }; + + public abstract NetworkParameters getParams(); + public abstract Collection getServers(); + public abstract String getGenesisHash(); + public abstract long getP2shFee(Long timestamp) throws ForeignBlockchainException; + } + + private static Bitcoin instance; + + private final BitcoinNet bitcoinNet; + + // Constructors and instance + + private Bitcoin(BitcoinNet bitcoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { + super(blockchain, bitcoinjContext, currencyCode); + this.bitcoinNet = bitcoinNet; + + LOGGER.info(() -> String.format("Starting Bitcoin support using %s", this.bitcoinNet.name())); + } + + public static synchronized Bitcoin getInstance() { + if (instance == null) { + BitcoinNet bitcoinNet = Settings.getInstance().getBitcoinNet(); + + BitcoinyBlockchainProvider electrumX = new ElectrumX("Bitcoin-" + bitcoinNet.name(), bitcoinNet.getGenesisHash(), bitcoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS); + Context bitcoinjContext = new Context(bitcoinNet.getParams()); + + instance = new Bitcoin(bitcoinNet, electrumX, bitcoinjContext, CURRENCY_CODE); + } + + return instance; + } + + // Getters & setters + + public static synchronized void resetForTesting() { + instance = null; + } + + // Actual useful methods for use by other classes + + /** + * Returns estimated BTC fee, in sats per 1000bytes, optionally for historic timestamp. + * + * @param timestamp optional milliseconds since epoch, or null for 'now' + * @return sats per 1000bytes, or throws ForeignBlockchainException if something went wrong + */ + @Override + public long getP2shFee(Long timestamp) throws ForeignBlockchainException { + return this.bitcoinNet.getP2shFee(timestamp); + } + +} diff --git a/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java b/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java new file mode 100644 index 000000000..eea541ad3 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java @@ -0,0 +1,922 @@ +package org.qortal.crosschain; + +import static org.ciyam.at.OpCode.calcOffset; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import org.ciyam.at.API; +import org.ciyam.at.CompilationException; +import org.ciyam.at.FunctionCode; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.ciyam.at.Timestamp; +import org.qortal.account.Account; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.utils.Base58; +import org.qortal.utils.BitTwiddling; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; + +/** + * Cross-chain trade AT + * + *

+ *

    + *
  • Bob generates Bitcoin & Qortal 'trade' keys, and secret-b + *
      + *
    • private key required to sign P2SH redeem tx
      • + *
    • private key could be used to create 'secret' (e.g. double-SHA256)
      • + *
    • encrypted private key could be stored in Qortal AT for access by Bob from any node
      • + *
    + *
    • + *
  • Bob deploys Qortal AT + *
      + *
    + *
    • + *
  • Alice finds Qortal AT and wants to trade + *
      + *
    • Alice generates Bitcoin & Qortal 'trade' keys
      • + *
    • Alice funds Bitcoin P2SH-A
      • + *
    • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: + *
        + *
      • hash-of-secret-A
        • + *
      • her 'trade' Bitcoin PKH
        • + *
      + *
      • + *
    + *
    • + *
  • Bob receives "offer" MESSAGE + *
      + *
    • Checks Alice's P2SH-A
      • + *
    • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: + *
        + *
      • Alice's trade Qortal address
        • + *
      • Alice's trade Bitcoin PKH
        • + *
      • hash-of-secret-A
        • + *
      + *
      • + *
    + *
    • + *
  • Alice checks Qortal AT to confirm it's locked to her + *
      + *
    • Alice creates/funds Bitcoin P2SH-B
      • + *
    + *
    • + *
  • Bob checks P2SH-B is funded + *
      + *
    • Bob redeems P2SH-B using his Bitcoin trade key and secret-B
      • + *
    + *
    • + *
  • Alice scans P2SH-B redeem transaction to extract secret-B + *
      + *
    • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: + *
        + *
      • secret-A
        • + *
      • secret-B
        • + *
      • Qortal receiving address of her chosing
        • + *
      + *
      • + *
    • AT's QORT funds are sent to Qortal receiving address
      • + *
    + *
    • + *
  • Bob checks AT, extracts secret-A + *
      + *
    • Bob redeems P2SH-A using his Bitcoin trade key and secret-A
      • + *
    • P2SH-A BTC funds end up at Bitcoin address determined by redeem transaction output(s)
      • + *
    + *
    • + *
+ */ +public class BitcoinACCTv1 implements ACCT { + + public static final String NAME = BitcoinACCTv1.class.getSimpleName(); + public static final byte[] CODE_BYTES_HASH = HashCode.fromString("f7f419522a9aaa3c671149878f8c1374dfc59d4fd86ca43ff2a4d913cfbc9e89").asBytes(); // SHA256 of AT code bytes + + public static final int SECRET_LENGTH = 32; + + /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ + private static final int MODE_VALUE_OFFSET = 68; + /** Byte offset into AT state data where 'mode' variable (long) is stored. */ + public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); + + public static class OfferMessageData { + public byte[] partnerBitcoinPKH; + public byte[] hashOfSecretA; + public long lockTimeA; + } + public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerBitcoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; + public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ + + 24 /*partner's Bitcoin PKH (padded from 20 to 24)*/ + + 8 /*lockTimeB*/ + + 24 /*hash of secret-A (padded from 20 to 24)*/ + + 8 /*lockTimeA*/; + public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret*/ + 32 /*secret*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; + public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; + + private static BitcoinACCTv1 instance; + + private BitcoinACCTv1() { + } + + public static synchronized BitcoinACCTv1 getInstance() { + if (instance == null) + instance = new BitcoinACCTv1(); + + return instance; + } + + @Override + public byte[] getCodeBytesHash() { + return CODE_BYTES_HASH; + } + + @Override + public int getModeByteOffset() { + return MODE_BYTE_OFFSET; + } + + @Override + public ForeignBlockchain getBlockchain() { + return Bitcoin.getInstance(); + } + + /** + * Returns Qortal AT creation bytes for cross-chain trading AT. + *

+ * tradeTimeout (minutes) is the time window for the trade partner to send the + * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. + * + * @param creatorTradeAddress AT creator's trade Qortal address, also used for refunds + * @param bitcoinPublicKeyHash 20-byte HASH160 of creator's trade Bitcoin public key + * @param hashOfSecretB 20-byte HASH160 of 32-byte secret-B + * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT + * @param bitcoinAmount how much BTC the AT creator is expecting to trade + * @param tradeTimeout suggested timeout for entire trade + */ + public static byte[] buildQortalAT(String creatorTradeAddress, byte[] bitcoinPublicKeyHash, byte[] hashOfSecretB, long qortAmount, long bitcoinAmount, int tradeTimeout) { + // Labels for data segment addresses + int addrCounter = 0; + + // Constants (with corresponding dataByteBuffer.put*() calls below) + + final int addrCreatorTradeAddress1 = addrCounter++; + final int addrCreatorTradeAddress2 = addrCounter++; + final int addrCreatorTradeAddress3 = addrCounter++; + final int addrCreatorTradeAddress4 = addrCounter++; + + final int addrBitcoinPublicKeyHash = addrCounter; + addrCounter += 4; + + final int addrHashOfSecretB = addrCounter; + addrCounter += 4; + + final int addrQortAmount = addrCounter++; + final int addrBitcoinAmount = addrCounter++; + final int addrTradeTimeout = addrCounter++; + + final int addrMessageTxnType = addrCounter++; + final int addrExpectedTradeMessageLength = addrCounter++; + final int addrExpectedRedeemMessageLength = addrCounter++; + + final int addrCreatorAddressPointer = addrCounter++; + final int addrHashOfSecretBPointer = addrCounter++; + final int addrQortalPartnerAddressPointer = addrCounter++; + final int addrMessageSenderPointer = addrCounter++; + + final int addrTradeMessagePartnerBitcoinPKHOffset = addrCounter++; + final int addrPartnerBitcoinPKHPointer = addrCounter++; + final int addrTradeMessageHashOfSecretAOffset = addrCounter++; + final int addrHashOfSecretAPointer = addrCounter++; + + final int addrRedeemMessageSecretBOffset = addrCounter++; + final int addrRedeemMessageReceivingAddressOffset = addrCounter++; + + final int addrMessageDataPointer = addrCounter++; + final int addrMessageDataLength = addrCounter++; + + final int addrPartnerReceivingAddressPointer = addrCounter++; + + final int addrEndOfConstants = addrCounter; + + // Variables + + final int addrCreatorAddress1 = addrCounter++; + final int addrCreatorAddress2 = addrCounter++; + final int addrCreatorAddress3 = addrCounter++; + final int addrCreatorAddress4 = addrCounter++; + + final int addrQortalPartnerAddress1 = addrCounter++; + final int addrQortalPartnerAddress2 = addrCounter++; + final int addrQortalPartnerAddress3 = addrCounter++; + final int addrQortalPartnerAddress4 = addrCounter++; + + final int addrLockTimeA = addrCounter++; + final int addrLockTimeB = addrCounter++; + final int addrRefundTimeout = addrCounter++; + final int addrRefundTimestamp = addrCounter++; + final int addrLastTxnTimestamp = addrCounter++; + final int addrBlockTimestamp = addrCounter++; + final int addrTxnType = addrCounter++; + final int addrResult = addrCounter++; + + final int addrMessageSender1 = addrCounter++; + final int addrMessageSender2 = addrCounter++; + final int addrMessageSender3 = addrCounter++; + final int addrMessageSender4 = addrCounter++; + + final int addrMessageLength = addrCounter++; + + final int addrMessageData = addrCounter; + addrCounter += 4; + + final int addrHashOfSecretA = addrCounter; + addrCounter += 4; + + final int addrPartnerBitcoinPKH = addrCounter; + addrCounter += 4; + + final int addrPartnerReceivingAddress = addrCounter; + addrCounter += 4; + + final int addrMode = addrCounter++; + assert addrMode == MODE_VALUE_OFFSET : "MODE_VALUE_OFFSET does not match addrMode"; + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // AT creator's trade Qortal address, decoded from Base58 + assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; + byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); + dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); + + // Bitcoin public key hash + assert dataByteBuffer.position() == addrBitcoinPublicKeyHash * MachineState.VALUE_SIZE : "addrBitcoinPublicKeyHash incorrect"; + dataByteBuffer.put(Bytes.ensureCapacity(bitcoinPublicKeyHash, 32, 0)); + + // Hash of secret-B + assert dataByteBuffer.position() == addrHashOfSecretB * MachineState.VALUE_SIZE : "addrHashOfSecretB incorrect"; + dataByteBuffer.put(Bytes.ensureCapacity(hashOfSecretB, 32, 0)); + + // Redeem Qort amount + assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; + dataByteBuffer.putLong(qortAmount); + + // Expected Bitcoin amount + assert dataByteBuffer.position() == addrBitcoinAmount * MachineState.VALUE_SIZE : "addrBitcoinAmount incorrect"; + dataByteBuffer.putLong(bitcoinAmount); + + // Suggested trade timeout (minutes) + assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; + dataByteBuffer.putLong(tradeTimeout); + + // We're only interested in MESSAGE transactions + assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; + dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); + + // Expected length of 'trade' MESSAGE data from AT creator + assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; + dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); + + // Expected length of 'redeem' MESSAGE data from trade partner + assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; + dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); + + // Index into data segment of AT creator's address, used by GET_B_IND + assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; + dataByteBuffer.putLong(addrCreatorAddress1); + + // Index into data segment of hash of secret B, used by GET_B_IND + assert dataByteBuffer.position() == addrHashOfSecretBPointer * MachineState.VALUE_SIZE : "addrHashOfSecretBPointer incorrect"; + dataByteBuffer.putLong(addrHashOfSecretB); + + // Index into data segment of partner's Qortal address, used by SET_B_IND + assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; + dataByteBuffer.putLong(addrQortalPartnerAddress1); + + // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND + assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; + dataByteBuffer.putLong(addrMessageSender1); + + // Offset into 'trade' MESSAGE data payload for extracting partner's Bitcoin PKH + assert dataByteBuffer.position() == addrTradeMessagePartnerBitcoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerBitcoinPKHOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Index into data segment of partner's Bitcoin PKH, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerBitcoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerBitcoinPKHPointer incorrect"; + dataByteBuffer.putLong(addrPartnerBitcoinPKH); + + // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A + assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; + dataByteBuffer.putLong(64L); + + // Index into data segment to hash of secret A, used by GET_B_IND + assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; + dataByteBuffer.putLong(addrHashOfSecretA); + + // Offset into 'redeem' MESSAGE data payload for extracting secret-B + assert dataByteBuffer.position() == addrRedeemMessageSecretBOffset * MachineState.VALUE_SIZE : "addrRedeemMessageSecretBOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address + assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; + dataByteBuffer.putLong(64L); + + // Source location and length for hashing any passed secret + assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; + dataByteBuffer.putLong(addrMessageData); + assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; + dataByteBuffer.putLong(32L); + + // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; + dataByteBuffer.putLong(addrPartnerReceivingAddress); + + assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; + + // Code labels + Integer labelRefund = null; + + Integer labelTradeTxnLoop = null; + Integer labelCheckTradeTxn = null; + Integer labelCheckCancelTxn = null; + Integer labelNotTradeNorCancelTxn = null; + Integer labelCheckNonRefundTradeTxn = null; + Integer labelTradeTxnExtract = null; + Integer labelRedeemTxnLoop = null; + Integer labelCheckRedeemTxn = null; + Integer labelCheckRedeemTxnSender = null; + Integer labelCheckSecretB = null; + Integer labelPayout = null; + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); + + // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ + + /* Transaction processing loop */ + labelTradeTxnLoop = codeByteBuffer.position(); + + // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckTradeTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); + + /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + // Message sender's address matches AT creator's trade address so go process 'trade' message + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); + + /* Checking message sender for possible cancel message */ + labelCheckCancelTxn = codeByteBuffer.position(); + + // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + // Partner address is AT creator's address, so cancel offer and finish. + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + /* Not trade nor cancel message */ + labelNotTradeNorCancelTxn = codeByteBuffer.position(); + + // Loop to find another transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Possible switch-to-trade-mode message */ + labelCheckNonRefundTradeTxn = codeByteBuffer.position(); + + // Check 'trade' message we received has expected number of message bytes + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to info extraction code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); + // Message length didn't match - go back to finding another 'trade' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Extracting info from 'trade' MESSAGE transaction */ + labelTradeTxnExtract = codeByteBuffer.position(); + + // Extract message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); + + // Extract trade partner's Bitcoin public key hash (PKH) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerBitcoinPKHOffset)); + // Extract partner's Bitcoin PKH (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerBitcoinPKHPointer)); + // Also extract lockTimeB + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeB)); + + // Grab next 32 bytes + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); + + // Extract hash-of-secret-a (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); + // Extract lockTimeA (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); + + // Calculate trade refund timeout: (lockTimeA - lockTimeB) / 2 / 60 + codeByteBuffer.put(OpCode.SET_DAT.compile(addrRefundTimeout, addrLockTimeA)); // refundTimeout = lockTimeA + codeByteBuffer.put(OpCode.SUB_DAT.compile(addrRefundTimeout, addrLockTimeB)); // refundTimeout -= lockTimeB + codeByteBuffer.put(OpCode.DIV_VAL.compile(addrRefundTimeout, 2L * 60L)); // refundTimeout /= 2 * 60 + // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); + + /* We are in 'trade mode' */ + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ + + // Fetch current block 'timestamp' + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); + // If we're not past refund 'timestamp' then look for next transaction + codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + // We're past refund 'timestamp' so go refund everything back to AT creator + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); + + /* Transaction processing loop */ + labelRedeemTxnLoop = codeByteBuffer.position(); + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckRedeemTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check message payload length */ + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to sender checking code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); + // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Check transaction's sender */ + labelCheckRedeemTxnSender = codeByteBuffer.position(); + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check 'secret-A' in transaction's message */ + + // Extract secret-A from first 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); + // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); + // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). + // Save the equality result (1 if they match, 0 otherwise) into addrResult. + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); + // If hashes don't match, addrResult will be zero so go find another transaction + codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckSecretB))); + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Check 'secret-B' in transaction's message */ + + labelCheckSecretB = codeByteBuffer.position(); + + // Extract secret-B from next 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageSecretBOffset)); + // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); + // Load B register with expected hash result (as pointed to by addrHashOfSecretBPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretBPointer)); + // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). + // Save the equality result (1 if they match, 0 otherwise) into addrResult. + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); + // If hashes don't match, addrResult will be zero so go find another transaction + codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Success! Pay arranged amount to receiving address */ + labelPayout = codeByteBuffer.position(); + + // Extract Qortal receiving address from next 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); + // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); + // Pay AT's balance to receiving address + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); + // Set redeemed mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + // Fall-through to refunding any remaining balance back to AT creator + + /* Refund balance back to AT creator */ + labelRefund = codeByteBuffer.position(); + + // Set refunded mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile BTC-QORT ACCT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + assert Arrays.equals(Crypto.digest(codeBytes), BitcoinACCTv1.CODE_BYTES_HASH) + : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { + ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + byte[] addressBytes = new byte[25]; // for general use + String atAddress = atStateData.getATAddress(); + + CrossChainTradeData tradeData = new CrossChainTradeData(); + + tradeData.foreignBlockchain = SupportedBlockchain.BITCOIN.name(); + tradeData.acctName = NAME; + + tradeData.qortalAtAddress = atAddress; + tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); + tradeData.creationTimestamp = creationTimestamp; + + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + + byte[] stateData = atStateData.getStateData(); + ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); + dataByteBuffer.position(MachineState.HEADER_LENGTH); + + /* Constants */ + + // Skip creator's trade address + dataByteBuffer.get(addressBytes); + tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Creator's Bitcoin/foreign public key hash + tradeData.creatorForeignPKH = new byte[20]; + dataByteBuffer.get(tradeData.creatorForeignPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes + + // Hash of secret B + tradeData.hashOfSecretB = new byte[20]; + dataByteBuffer.get(tradeData.hashOfSecretB); + dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.hashOfSecretB.length); // skip to 32 bytes + + // Redeem payout + tradeData.qortAmount = dataByteBuffer.getLong(); + + // Expected BTC amount + tradeData.expectedForeignAmount = dataByteBuffer.getLong(); + + // Trade timeout + tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); + + // Skip MESSAGE transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'trade' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'redeem' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to hash-of-secret-B + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Qortal trade address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message sender + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for partner's bitcoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's bitcoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'redeem' message data offset for secret-B + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'redeem' message data offset for partner's Qortal receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message data + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip message data length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + /* End of constants / begin variables */ + + // Skip AT creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Partner's trade address (if present) + dataByteBuffer.get(addressBytes); + String qortalRecipient = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Potential lockTimeA (if in trade mode) + int lockTimeA = (int) dataByteBuffer.getLong(); + + // Potential lockTimeB (if in trade mode) + int lockTimeB = (int) dataByteBuffer.getLong(); + + // AT refund timeout (probably only useful for debugging) + int refundTimeout = (int) dataByteBuffer.getLong(); + + // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) + long tradeRefundTimestamp = dataByteBuffer.getLong(); + + // Skip last transaction timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip block timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary result + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message sender + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Skip message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message data + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Potential hash160 of secret A + byte[] hashOfSecretA = new byte[20]; + dataByteBuffer.get(hashOfSecretA); + dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes + + // Potential partner's Bitcoin PKH + byte[] partnerBitcoinPKH = new byte[20]; + dataByteBuffer.get(partnerBitcoinPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerBitcoinPKH.length); // skip to 32 bytes + + // Partner's receiving address (if present) + byte[] partnerReceivingAddress = new byte[25]; + dataByteBuffer.get(partnerReceivingAddress); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes + + // Trade AT's 'mode' + long modeValue = dataByteBuffer.getLong(); + AcctMode acctMode = AcctMode.valueOf((int) (modeValue & 0xffL)); + + /* End of variables */ + + if (acctMode != null && acctMode != AcctMode.OFFERING) { + tradeData.mode = acctMode; + tradeData.refundTimeout = refundTimeout; + tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; + tradeData.qortalPartnerAddress = qortalRecipient; + tradeData.hashOfSecretA = hashOfSecretA; + tradeData.partnerForeignPKH = partnerBitcoinPKH; + tradeData.lockTimeA = lockTimeA; + tradeData.lockTimeB = lockTimeB; + + if (acctMode == AcctMode.REDEEMED) + tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); + } else { + tradeData.mode = AcctMode.OFFERING; + } + + tradeData.duplicateDeprecated(); + + return tradeData; + } + + /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ + public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); + } + + /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ + public static OfferMessageData extractOfferMessageData(byte[] messageData) { + if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) + return null; + + OfferMessageData offerMessageData = new OfferMessageData(); + offerMessageData.partnerBitcoinPKH = Arrays.copyOfRange(messageData, 0, 20); + offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); + offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); + + return offerMessageData; + } + + /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ + public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int lockTimeB) { + byte[] data = new byte[TRADE_MESSAGE_LENGTH]; + byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + byte[] lockTimeBBytes = BitTwiddling.toBEByteArray((long) lockTimeB); + + System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); + System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); + System.arraycopy(lockTimeBBytes, 0, data, 56, lockTimeBBytes.length); + System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); + System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); + + return data; + } + + /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ + @Override + public byte[] buildCancelMessage(String creatorQortalAddress) { + byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; + byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); + + System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); + + return data; + } + + /** Returns 'redeem' MESSAGE payload for trade partner/ to send to AT. */ + public static byte[] buildRedeemMessage(byte[] secretA, byte[] secretB, String qortalReceivingAddress) { + byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; + byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); + + System.arraycopy(secretA, 0, data, 0, secretA.length); + System.arraycopy(secretB, 0, data, 32, secretB.length); + System.arraycopy(qortalReceivingAddressBytes, 0, data, 64, qortalReceivingAddressBytes.length); + + return data; + } + + /** Returns P2SH-B lockTime (epoch seconds) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ + public static int calcLockTimeB(long offerMessageTimestamp, int lockTimeA) { + // lockTimeB is halfway between offerMessageTimestamp and lockTimeA + return (int) ((lockTimeA + (offerMessageTimestamp / 1000L)) / 2L); + } + + @Override + public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { + String atAddress = crossChainTradeData.qortalAtAddress; + String redeemerAddress = crossChainTradeData.qortalPartnerAddress; + + // We don't have partner's public key so we check every message to AT + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); + if (messageTransactionsData == null) + return null; + + // Find 'redeem' message + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + // Check message payload type/encryption + if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) + continue; + + // Check message payload size + byte[] messageData = messageTransactionData.getData(); + if (messageData.length != REDEEM_MESSAGE_LENGTH) + // Wrong payload length + continue; + + // Check sender + if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) + // Wrong sender; + continue; + + // Extract both secretA & secretB + byte[] secretA = new byte[32]; + System.arraycopy(messageData, 0, secretA, 0, secretA.length); + byte[] secretB = new byte[32]; + System.arraycopy(messageData, 32, secretB, 0, secretB.length); + + byte[] hashOfSecretA = Crypto.hash160(secretA); + if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) + continue; + + byte[] hashOfSecretB = Crypto.hash160(secretB); + if (!Arrays.equals(hashOfSecretB, crossChainTradeData.hashOfSecretB)) + continue; + + return secretA; + } + + return null; + } + +} diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java new file mode 100644 index 000000000..bc6f79e14 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -0,0 +1,863 @@ +package org.qortal.crosschain; + +import java.util.*; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.Address; +import org.bitcoinj.core.AddressFormatException; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.Context; +import org.bitcoinj.core.ECKey; +import org.bitcoinj.core.InsufficientMoneyException; +import org.bitcoinj.core.LegacyAddress; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.core.Sha256Hash; +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.TransactionOutput; +import org.bitcoinj.core.UTXO; +import org.bitcoinj.core.UTXOProvider; +import org.bitcoinj.core.UTXOProviderException; +import org.bitcoinj.crypto.ChildNumber; +import org.bitcoinj.crypto.DeterministicHierarchy; +import org.bitcoinj.crypto.DeterministicKey; +import org.bitcoinj.script.Script.ScriptType; +import org.bitcoinj.script.ScriptBuilder; +import org.bitcoinj.wallet.DeterministicKeyChain; +import org.bitcoinj.wallet.SendRequest; +import org.bitcoinj.wallet.Wallet; +import org.qortal.api.model.SimpleForeignTransaction; +import org.qortal.crypto.Crypto; +import org.qortal.utils.Amounts; +import org.qortal.utils.BitTwiddling; + +import com.google.common.hash.HashCode; +import org.qortal.utils.NTP; + +/** Bitcoin-like (Bitcoin, Litecoin, etc.) support */ +public abstract class Bitcoiny implements ForeignBlockchain { + + protected static final Logger LOGGER = LogManager.getLogger(Bitcoiny.class); + + public static final int HASH160_LENGTH = 20; + + protected final BitcoinyBlockchainProvider blockchain; + protected final Context bitcoinjContext; + protected final String currencyCode; + + protected final NetworkParameters params; + + /** Cache recent transactions to speed up subsequent lookups */ + protected List transactionsCache; + protected Long transactionsCacheTimestamp; + protected String transactionsCacheXpub; + protected static long TRANSACTIONS_CACHE_TIMEOUT = 2 * 60 * 1000L; // 2 minutes + + /** Keys that have been previously marked as fully spent,
+ * i.e. keys with transactions but with no unspent outputs. */ + protected final Set spentKeys = Collections.synchronizedSet(new HashSet<>()); + + /** How many wallet keys to generate in each batch. */ + private static final int WALLET_KEY_LOOKAHEAD_INCREMENT = 3; + + /** How many wallet keys to generate when using bitcoinj as the data provider. + * We must use a higher value here since we are unable to request multiple batches of keys. + * Without this, the bitcoinj state can be missing transactions, causing errors such as "insufficient balance". */ + private static final int WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ = 50; + + /** Byte offset into raw block headers to block timestamp. */ + private static final int TIMESTAMP_OFFSET = 4 + 32 + 32; + + // Constructors and instance + + protected Bitcoiny(BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { + this.blockchain = blockchain; + this.bitcoinjContext = bitcoinjContext; + this.currencyCode = currencyCode; + + this.params = this.bitcoinjContext.getParams(); + } + + // Getters & setters + + public BitcoinyBlockchainProvider getBlockchainProvider() { + return this.blockchain; + } + + public Context getBitcoinjContext() { + return this.bitcoinjContext; + } + + public String getCurrencyCode() { + return this.currencyCode; + } + + public NetworkParameters getNetworkParameters() { + return this.params; + } + + // Interface obligations + + @Override + public boolean isValidAddress(String address) { + try { + ScriptType addressType = Address.fromString(this.params, address).getOutputScriptType(); + + return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH; + } catch (AddressFormatException e) { + return false; + } + } + + @Override + public boolean isValidWalletKey(String walletKey) { + return this.isValidDeterministicKey(walletKey); + } + + // Actual useful methods for use by other classes + + public String format(Coin amount) { + return this.format(amount.value); + } + + public String format(long amount) { + return Amounts.prettyAmount(amount) + " " + this.currencyCode; + } + + public boolean isValidDeterministicKey(String key58) { + try { + Context.propagate(this.bitcoinjContext); + DeterministicKey.deserializeB58(null, key58, this.params); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + + /** Returns P2PKH address using passed public key hash. */ + public String pkhToAddress(byte[] publicKeyHash) { + Context.propagate(this.bitcoinjContext); + return LegacyAddress.fromPubKeyHash(this.params, publicKeyHash).toString(); + } + + /** Returns P2SH address using passed redeem script. */ + public String deriveP2shAddress(byte[] redeemScriptBytes) { + Context.propagate(bitcoinjContext); + byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + return LegacyAddress.fromScriptHash(this.params, redeemScriptHash).toString(); + } + + /** + * Returns median timestamp from latest 11 blocks, in seconds. + *

+ * @throws ForeignBlockchainException if error occurs + */ + public int getMedianBlockTime() throws ForeignBlockchainException { + int height = this.blockchain.getCurrentHeight(); + + // Grab latest 11 blocks + List blockHeaders = this.blockchain.getRawBlockHeaders(height - 11, 11); + if (blockHeaders.size() < 11) + throw new ForeignBlockchainException("Not enough blocks to determine median block time"); + + List blockTimestamps = blockHeaders.stream().map(blockHeader -> BitTwiddling.intFromLEBytes(blockHeader, TIMESTAMP_OFFSET)).collect(Collectors.toList()); + + // Descending order + blockTimestamps.sort((a, b) -> Integer.compare(b, a)); + + // Pick median + return blockTimestamps.get(5); + } + + /** Returns fee per transaction KB. To be overridden for testnet/regtest. */ + public Coin getFeePerKb() { + return this.bitcoinjContext.getFeePerKb(); + } + + /** Returns minimum order size in sats. To be overridden for coins that need to restrict order size. */ + public long getMinimumOrderAmount() { + return 0L; + } + + /** + * Returns fixed P2SH spending fee, in sats per 1000bytes, optionally for historic timestamp. + * + * @param timestamp optional milliseconds since epoch, or null for 'now' + * @return sats per 1000bytes + * @throws ForeignBlockchainException if something went wrong + */ + public abstract long getP2shFee(Long timestamp) throws ForeignBlockchainException; + + /** + * Returns confirmed balance, based on passed payment script. + *

+ * @return confirmed balance, or zero if script unknown + * @throws ForeignBlockchainException if there was an error + */ + public long getConfirmedBalance(String base58Address) throws ForeignBlockchainException { + return this.blockchain.getConfirmedBalance(addressToScriptPubKey(base58Address)); + } + + /** + * Returns list of unspent outputs pertaining to passed address. + *

+ * @return list of unspent outputs, or empty list if address unknown + * @throws ForeignBlockchainException if there was an error. + */ + // TODO: don't return bitcoinj-based objects like TransactionOutput, use BitcoinyTransaction.Output instead + public List getUnspentOutputs(String base58Address) throws ForeignBlockchainException { + List unspentOutputs = this.blockchain.getUnspentOutputs(addressToScriptPubKey(base58Address), false); + + List unspentTransactionOutputs = new ArrayList<>(); + for (UnspentOutput unspentOutput : unspentOutputs) { + List transactionOutputs = this.getOutputs(unspentOutput.hash); + + unspentTransactionOutputs.add(transactionOutputs.get(unspentOutput.index)); + } + + return unspentTransactionOutputs; + } + + /** + * Returns list of outputs pertaining to passed transaction hash. + *

+ * @return list of outputs, or empty list if transaction unknown + * @throws ForeignBlockchainException if there was an error. + */ + // TODO: don't return bitcoinj-based objects like TransactionOutput, use BitcoinyTransaction.Output instead + public List getOutputs(byte[] txHash) throws ForeignBlockchainException { + byte[] rawTransactionBytes = this.blockchain.getRawTransaction(txHash); + + Context.propagate(bitcoinjContext); + Transaction transaction = new Transaction(this.params, rawTransactionBytes); + return transaction.getOutputs(); + } + + /** + * Returns transactions for passed script + *

+ * @throws ForeignBlockchainException if error occurs + */ + public List getAddressTransactions(byte[] scriptPubKey, boolean includeUnconfirmed) throws ForeignBlockchainException { + int retries = 0; + ForeignBlockchainException e2 = null; + while (retries <= 3) { + try { + return this.blockchain.getAddressTransactions(scriptPubKey, includeUnconfirmed); + } catch (ForeignBlockchainException e) { + e2 = e; + retries++; + } + } + throw(e2); + } + + /** + * Returns list of transaction hashes pertaining to passed address. + *

+ * @return list of unspent outputs, or empty list if script unknown + * @throws ForeignBlockchainException if there was an error. + */ + public List getAddressTransactions(String base58Address, boolean includeUnconfirmed) throws ForeignBlockchainException { + return this.blockchain.getAddressTransactions(addressToScriptPubKey(base58Address), includeUnconfirmed); + } + + /** + * Returns list of raw, confirmed transactions involving given address. + *

+ * @throws ForeignBlockchainException if there was an error + */ + public List getAddressTransactions(String base58Address) throws ForeignBlockchainException { + List transactionHashes = this.blockchain.getAddressTransactions(addressToScriptPubKey(base58Address), false); + + List rawTransactions = new ArrayList<>(); + for (TransactionHash transactionInfo : transactionHashes) { + byte[] rawTransaction = this.blockchain.getRawTransaction(HashCode.fromString(transactionInfo.txHash).asBytes()); + rawTransactions.add(rawTransaction); + } + + return rawTransactions; + } + + /** + * Returns transaction info for passed transaction hash. + *

+ * @throws ForeignBlockchainException.NotFoundException if transaction unknown + * @throws ForeignBlockchainException if error occurs + */ + public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException { + int retries = 0; + ForeignBlockchainException e2 = null; + while (retries <= 3) { + try { + return this.blockchain.getTransaction(txHash); + } catch (ForeignBlockchainException e) { + e2 = e; + retries++; + } + } + throw(e2); + } + + /** + * Broadcasts raw transaction to network. + *

+ * @throws ForeignBlockchainException if error occurs + */ + public void broadcastTransaction(Transaction transaction) throws ForeignBlockchainException { + this.blockchain.broadcastTransaction(transaction.bitcoinSerialize()); + } + + /** + * Returns bitcoinj transaction sending amount to recipient. + * + * @param xprv58 BIP32 private key + * @param recipient P2PKH address + * @param amount unscaled amount + * @param feePerByte unscaled fee per byte, or null to use default fees + * @return transaction, or null if insufficient funds + */ + public Transaction buildSpend(String xprv58, String recipient, long amount, Long feePerByte) { + Context.propagate(bitcoinjContext); + + Wallet wallet = Wallet.fromSpendingKeyB58(this.params, xprv58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS); + wallet.setUTXOProvider(new WalletAwareUTXOProvider(this, wallet)); + + Address destination = Address.fromString(this.params, recipient); + SendRequest sendRequest = SendRequest.to(destination, Coin.valueOf(amount)); + + if (feePerByte != null) + sendRequest.feePerKb = Coin.valueOf(feePerByte * 1000L); // Note: 1000 not 1024 + else + // Allow override of default for TestNet3, etc. + sendRequest.feePerKb = this.getFeePerKb(); + + try { + wallet.completeTx(sendRequest); + return sendRequest.tx; + } catch (InsufficientMoneyException e) { + return null; + } + } + + /** + * Returns bitcoinj transaction sending amount to recipient using default fees. + * + * @param xprv58 BIP32 private key + * @param recipient P2PKH address + * @param amount unscaled amount + * @return transaction, or null if insufficient funds + */ + public Transaction buildSpend(String xprv58, String recipient, long amount) { + return buildSpend(xprv58, recipient, amount, null); + } + + /** + * Returns unspent Bitcoin balance given 'm' BIP32 key. + * + * @param key58 BIP32/HD extended Bitcoin private/public key + * @return unspent BTC balance, or null if unable to determine balance + */ + public Long getWalletBalance(String key58) { + Context.propagate(bitcoinjContext); + + Wallet wallet = walletFromDeterministicKey58(key58); + wallet.setUTXOProvider(new WalletAwareUTXOProvider(this, wallet)); + + Coin balance = wallet.getBalance(); + if (balance == null) + return null; + + return balance.value; + } + + public Long getWalletBalanceFromTransactions(String key58) throws ForeignBlockchainException { + long balance = 0; + Comparator oldestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp); + List transactions = getWalletTransactions(key58).stream().sorted(oldestTimestampFirstComparator).collect(Collectors.toList()); + for (SimpleTransaction transaction : transactions) { + balance += transaction.getTotalAmount(); + } + return balance; + } + + public List getWalletTransactions(String key58) throws ForeignBlockchainException { + synchronized (this) { + // Serve from the cache if it's recent, and matches this xpub + if (Objects.equals(transactionsCacheXpub, key58)) { + if (transactionsCache != null && transactionsCacheTimestamp != null) { + Long now = NTP.getTime(); + boolean isCacheStale = (now != null && now - transactionsCacheTimestamp >= TRANSACTIONS_CACHE_TIMEOUT); + if (!isCacheStale) { + return transactionsCache; + } + } + } + + Context.propagate(bitcoinjContext); + + Wallet wallet = walletFromDeterministicKey58(key58); + DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); + + keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + keyChain.maybeLookAhead(); + + List keys = new ArrayList<>(keyChain.getLeafKeys()); + + Set walletTransactions = new HashSet<>(); + Set keySet = new HashSet<>(); + + // Set the number of consecutive empty batches required before giving up + final int numberOfAdditionalBatchesToSearch = 7; + + int unusedCounter = 0; + int ki = 0; + do { + boolean areAllKeysUnused = true; + + for (; ki < keys.size(); ++ki) { + DeterministicKey dKey = keys.get(ki); + + // Check for transactions + Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); + keySet.add(address.toString()); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + + // Ask for transaction history - if it's empty then key has never been used + List historicTransactionHashes = this.getAddressTransactions(script, false); + + if (!historicTransactionHashes.isEmpty()) { + areAllKeysUnused = false; + + for (TransactionHash transactionHash : historicTransactionHashes) + walletTransactions.add(this.getTransaction(transactionHash.txHash)); + } + } + + if (areAllKeysUnused) { + // No transactions + if (unusedCounter >= numberOfAdditionalBatchesToSearch) { + // ... and we've hit our search limit + break; + } + // We haven't hit our search limit yet so increment the counter and keep looking + unusedCounter++; + } else { + // Some keys in this batch were used, so reset the counter + unusedCounter = 0; + } + + // Generate some more keys + keys.addAll(generateMoreKeys(keyChain)); + + // Process new keys + } while (true); + + Comparator newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed(); + + // Update cache and return + transactionsCacheTimestamp = NTP.getTime(); + transactionsCacheXpub = key58; + transactionsCache = walletTransactions.stream() + .map(t -> convertToSimpleTransaction(t, keySet)) + .sorted(newestTimestampFirstComparator).collect(Collectors.toList()); + + return transactionsCache; + } + } + + protected SimpleTransaction convertToSimpleTransaction(BitcoinyTransaction t, Set keySet) { + long amount = 0; + long total = 0L; + long totalInputAmount = 0L; + long totalOutputAmount = 0L; + List inputs = new ArrayList<>(); + List outputs = new ArrayList<>(); + + boolean anyOutputAddressInWallet = false; + boolean transactionInvolvesExternalWallet = false; + + for (BitcoinyTransaction.Input input : t.inputs) { + try { + BitcoinyTransaction t2 = getTransaction(input.outputTxHash); + List senders = t2.outputs.get(input.outputVout).addresses; + long inputAmount = t2.outputs.get(input.outputVout).value; + totalInputAmount += inputAmount; + if (senders != null) { + for (String sender : senders) { + boolean addressInWallet = false; + if (keySet.contains(sender)) { + total += inputAmount; + addressInWallet = true; + } + else { + transactionInvolvesExternalWallet = true; + } + inputs.add(new SimpleTransaction.Input(sender, inputAmount, addressInWallet)); + } + } + } catch (ForeignBlockchainException e) { + LOGGER.trace("Failed to retrieve transaction information {}", input.outputTxHash); + } + } + if (t.outputs != null && !t.outputs.isEmpty()) { + for (BitcoinyTransaction.Output output : t.outputs) { + if (output.addresses != null) { + for (String address : output.addresses) { + boolean addressInWallet = false; + if (keySet.contains(address)) { + if (total > 0L) { // Change returned from sent amount + amount -= (total - output.value); + } else { // Amount received + amount += output.value; + } + addressInWallet = true; + anyOutputAddressInWallet = true; + } + else { + transactionInvolvesExternalWallet = true; + } + outputs.add(new SimpleTransaction.Output(address, output.value, addressInWallet)); + } + } + totalOutputAmount += output.value; + } + } + long fee = totalInputAmount - totalOutputAmount; + + if (!anyOutputAddressInWallet) { + // No outputs relate to this wallet - check if any inputs did (which is signified by a positive total) + if (total > 0) { + amount = total * -1; + } + } + else if (!transactionInvolvesExternalWallet) { + // All inputs and outputs relate to this wallet, so the balance should be unaffected + amount = 0; + } + return new SimpleTransaction(t.txHash, t.timestamp, amount, fee, inputs, outputs); + } + + /** + * Returns first unused receive address given 'm' BIP32 key. + * + * @param key58 BIP32/HD extended Bitcoin private/public key + * @return P2PKH address + * @throws ForeignBlockchainException if something went wrong + */ + public String getUnusedReceiveAddress(String key58) throws ForeignBlockchainException { + Context.propagate(bitcoinjContext); + + Wallet wallet = walletFromDeterministicKey58(key58); + DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); + + keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + keyChain.maybeLookAhead(); + + final int keyChainPathSize = keyChain.getAccountPath().size(); + List keys = new ArrayList<>(keyChain.getLeafKeys()); + + int ki = 0; + do { + for (; ki < keys.size(); ++ki) { + DeterministicKey dKey = keys.get(ki); + List dKeyPath = dKey.getPath(); + + // If keyChain is based on 'm', then make sure dKey is m/0/ki - i.e. a 'receive' address, not 'change' (m/1/ki) + if (dKeyPath.size() != keyChainPathSize + 2 || dKeyPath.get(dKeyPath.size() - 2) != ChildNumber.ZERO) + continue; + + // Check unspent + Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + + List unspentOutputs = this.blockchain.getUnspentOutputs(script, false); + + /* + * If there are no unspent outputs then either: + * a) all the outputs have been spent + * b) address has never been used + * + * For case (a) we want to remember not to check this address (key) again. + */ + + if (unspentOutputs.isEmpty()) { + // If this is a known key that has been spent before, then we can skip asking for transaction history + if (this.spentKeys.contains(dKey)) { + wallet.getActiveKeyChain().markKeyAsUsed(dKey); + continue; + } + + // Ask for transaction history - if it's empty then key has never been used + List historicTransactionHashes = this.blockchain.getAddressTransactions(script, false); + + if (!historicTransactionHashes.isEmpty()) { + // Fully spent key - case (a) + this.spentKeys.add(dKey); + wallet.getActiveKeyChain().markKeyAsUsed(dKey); + continue; + } + + // Key never been used - case (b) + return address.toString(); + } + + // Key has unspent outputs, hence used, so no good to us + this.spentKeys.remove(dKey); + } + + // Generate some more keys + keys.addAll(generateMoreKeys(keyChain)); + + // Process new keys + } while (true); + } + + // UTXOProvider support + + static class WalletAwareUTXOProvider implements UTXOProvider { + private final Bitcoiny bitcoiny; + private final Wallet wallet; + + private final DeterministicKeyChain keyChain; + + public WalletAwareUTXOProvider(Bitcoiny bitcoiny, Wallet wallet) { + this.bitcoiny = bitcoiny; + this.wallet = wallet; + this.keyChain = this.wallet.getActiveKeyChain(); + + // Set up wallet's key chain + this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ); + this.keyChain.maybeLookAhead(); + } + + @Override + public List getOpenTransactionOutputs(List keys) throws UTXOProviderException { + List allUnspentOutputs = new ArrayList<>(); + final boolean coinbase = false; + + int ki = 0; + do { + boolean areAllKeysUnspent = true; + + for (; ki < keys.size(); ++ki) { + ECKey key = keys.get(ki); + + Address address = Address.fromKey(this.bitcoiny.params, key, ScriptType.P2PKH); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + + List unspentOutputs; + try { + unspentOutputs = this.bitcoiny.blockchain.getUnspentOutputs(script, false); + } catch (ForeignBlockchainException e) { + throw new UTXOProviderException(String.format("Unable to fetch unspent outputs for %s", address)); + } + + /* + * If there are no unspent outputs then either: + * a) all the outputs have been spent + * b) address has never been used + * + * For case (a) we want to remember not to check this address (key) again. + */ + + if (unspentOutputs.isEmpty()) { + // If this is a known key that has been spent before, then we can skip asking for transaction history + if (this.bitcoiny.spentKeys.contains(key)) { + this.wallet.getActiveKeyChain().markKeyAsUsed((DeterministicKey) key); + areAllKeysUnspent = false; + continue; + } + + // Ask for transaction history - if it's empty then key has never been used + List historicTransactionHashes; + try { + historicTransactionHashes = this.bitcoiny.blockchain.getAddressTransactions(script, false); + } catch (ForeignBlockchainException e) { + throw new UTXOProviderException(String.format("Unable to fetch transaction history for %s", address)); + } + + if (!historicTransactionHashes.isEmpty()) { + // Fully spent key - case (a) + this.bitcoiny.spentKeys.add(key); + this.wallet.getActiveKeyChain().markKeyAsUsed((DeterministicKey) key); + areAllKeysUnspent = false; + } else { + // Key never been used - case (b) + } + + continue; + } + + // If we reach here, then there's definitely at least one unspent key + this.bitcoiny.spentKeys.remove(key); + + for (UnspentOutput unspentOutput : unspentOutputs) { + List transactionOutputs; + try { + transactionOutputs = this.bitcoiny.getOutputs(unspentOutput.hash); + } catch (ForeignBlockchainException e) { + throw new UTXOProviderException(String.format("Unable to fetch outputs for TX %s", + HashCode.fromBytes(unspentOutput.hash))); + } + + TransactionOutput transactionOutput = transactionOutputs.get(unspentOutput.index); + + UTXO utxo = new UTXO(Sha256Hash.wrap(unspentOutput.hash), unspentOutput.index, + Coin.valueOf(unspentOutput.value), unspentOutput.height, coinbase, + transactionOutput.getScriptPubKey()); + + allUnspentOutputs.add(utxo); + } + } + + if (areAllKeysUnspent) + // No transactions for this batch of keys so assume we're done searching. + return allUnspentOutputs; + + // Generate some more keys + keys.addAll(Bitcoiny.generateMoreKeys(this.keyChain)); + + // Process new keys + } while (true); + } + + @Override + public int getChainHeadHeight() throws UTXOProviderException { + try { + return this.bitcoiny.blockchain.getCurrentHeight(); + } catch (ForeignBlockchainException e) { + throw new UTXOProviderException("Unable to determine Bitcoiny chain height"); + } + } + + @Override + public NetworkParameters getParams() { + return this.bitcoiny.params; + } + } + + // Utility methods for others + + public static List simplifyWalletTransactions(List transactions) { + // Sort by oldest timestamp first + transactions.sort(Comparator.comparingInt(t -> t.timestamp)); + + // Manual 2nd-level sort same-timestamp transactions so that a transaction's input comes first + int fromIndex = 0; + do { + int timestamp = transactions.get(fromIndex).timestamp; + + int toIndex; + for (toIndex = fromIndex + 1; toIndex < transactions.size(); ++toIndex) + if (transactions.get(toIndex).timestamp != timestamp) + break; + + // Process same-timestamp sub-list + List subList = transactions.subList(fromIndex, toIndex); + + // Only if necessary + if (subList.size() > 1) { + // Quick index lookup + Map indexByTxHash = subList.stream().collect(Collectors.toMap(t -> t.txHash, t -> t.timestamp)); + + int restartIndex = 0; + boolean isSorted; + do { + isSorted = true; + + for (int ourIndex = restartIndex; ourIndex < subList.size(); ++ourIndex) { + BitcoinyTransaction ourTx = subList.get(ourIndex); + + for (BitcoinyTransaction.Input input : ourTx.inputs) { + Integer inputIndex = indexByTxHash.get(input.outputTxHash); + + if (inputIndex != null && inputIndex > ourIndex) { + // Input tx is currently after current tx, so swap + BitcoinyTransaction tmpTx = subList.get(inputIndex); + subList.set(inputIndex, ourTx); + subList.set(ourIndex, tmpTx); + + // Update index lookup too + indexByTxHash.put(ourTx.txHash, inputIndex); + indexByTxHash.put(tmpTx.txHash, ourIndex); + + if (isSorted) + restartIndex = Math.max(restartIndex, ourIndex); + + isSorted = false; + break; + } + } + } + } while (!isSorted); + } + + fromIndex = toIndex; + } while (fromIndex < transactions.size()); + + // Simplify + List simpleTransactions = new ArrayList<>(); + + // Quick lookup of txs in our wallet + Set walletTxHashes = transactions.stream().map(t -> t.txHash).collect(Collectors.toSet()); + + for (BitcoinyTransaction transaction : transactions) { + SimpleForeignTransaction.Builder builder = new SimpleForeignTransaction.Builder(); + builder.txHash(transaction.txHash); + builder.timestamp(transaction.timestamp); + + builder.isSentNotReceived(false); + + for (BitcoinyTransaction.Input input : transaction.inputs) { + // TODO: add input via builder + + if (walletTxHashes.contains(input.outputTxHash)) + builder.isSentNotReceived(true); + } + + for (BitcoinyTransaction.Output output : transaction.outputs) + builder.output(output.addresses, output.value); + + simpleTransactions.add(builder.build()); + } + + return simpleTransactions; + } + + // Utility methods for us + + protected static List generateMoreKeys(DeterministicKeyChain keyChain) { + int existingLeafKeyCount = keyChain.getLeafKeys().size(); + + // Increase lookahead size... + keyChain.setLookaheadSize(keyChain.getLookaheadSize() + Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + // ...and lookahead threshold (minimum number of keys to generate)... + keyChain.setLookaheadThreshold(0); + // ...so that this call will generate more keys + keyChain.maybeLookAhead(); + + // This returns *all* keys + List allLeafKeys = keyChain.getLeafKeys(); + + // Only return newly generated keys + return allLeafKeys.subList(existingLeafKeyCount, allLeafKeys.size()); + } + + protected byte[] addressToScriptPubKey(String base58Address) { + Context.propagate(this.bitcoinjContext); + Address address = Address.fromString(this.params, base58Address); + return ScriptBuilder.createOutputScript(address).getProgram(); + } + + protected Wallet walletFromDeterministicKey58(String key58) { + DeterministicKey dKey = DeterministicKey.deserializeB58(null, key58, this.params); + + if (dKey.hasPrivKey()) + return Wallet.fromSpendingKeyB58(this.params, key58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS); + else + return Wallet.fromWatchingKeyB58(this.params, key58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS); + } + +} diff --git a/src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java b/src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java new file mode 100644 index 000000000..7691efb12 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java @@ -0,0 +1,40 @@ +package org.qortal.crosschain; + +import java.util.List; + +public abstract class BitcoinyBlockchainProvider { + + public static final boolean INCLUDE_UNCONFIRMED = true; + public static final boolean EXCLUDE_UNCONFIRMED = false; + + /** Returns ID unique to bitcoiny network (e.g. "Litecoin-TEST3") */ + public abstract String getNetId(); + + /** Returns current blockchain height. */ + public abstract int getCurrentHeight() throws ForeignBlockchainException; + + /** Returns a list of raw block headers, starting at startHeight (inclusive), up to count max. */ + public abstract List getRawBlockHeaders(int startHeight, int count) throws ForeignBlockchainException; + + /** Returns balance of address represented by scriptPubKey. */ + public abstract long getConfirmedBalance(byte[] scriptPubKey) throws ForeignBlockchainException; + + /** Returns raw, serialized, transaction bytes given txHash. */ + public abstract byte[] getRawTransaction(String txHash) throws ForeignBlockchainException; + + /** Returns raw, serialized, transaction bytes given txHash. */ + public abstract byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException; + + /** Returns unpacked transaction given txHash. */ + public abstract BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException; + + /** Returns list of transaction hashes (and heights) for address represented by scriptPubKey, optionally including unconfirmed transactions. */ + public abstract List getAddressTransactions(byte[] scriptPubKey, boolean includeUnconfirmed) throws ForeignBlockchainException; + + /** Returns list of unspent transaction outputs for address represented by scriptPubKey, optionally including unconfirmed transactions. */ + public abstract List getUnspentOutputs(byte[] scriptPubKey, boolean includeUnconfirmed) throws ForeignBlockchainException; + + /** Broadcasts raw, serialized, transaction bytes to network, returning success/failure. */ + public abstract void broadcastTransaction(byte[] rawTransaction) throws ForeignBlockchainException; + +} diff --git a/src/main/java/org/qortal/crosschain/BitcoinyHTLC.java b/src/main/java/org/qortal/crosschain/BitcoinyHTLC.java new file mode 100644 index 000000000..8ebfffa27 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/BitcoinyHTLC.java @@ -0,0 +1,438 @@ +package org.qortal.crosschain; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.ECKey; +import org.bitcoinj.core.LegacyAddress; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.Transaction.SigHash; +import org.bitcoinj.core.TransactionInput; +import org.bitcoinj.core.TransactionOutput; +import org.bitcoinj.crypto.TransactionSignature; +import org.bitcoinj.script.Script; +import org.bitcoinj.script.ScriptBuilder; +import org.bitcoinj.script.ScriptChunk; +import org.bitcoinj.script.ScriptOpCodes; +import org.qortal.crypto.Crypto; +import org.qortal.utils.Base58; +import org.qortal.utils.BitTwiddling; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; + +public class BitcoinyHTLC { + + public enum Status { + UNFUNDED, FUNDING_IN_PROGRESS, FUNDED, REDEEM_IN_PROGRESS, REDEEMED, REFUND_IN_PROGRESS, REFUNDED + } + + public static final int SECRET_LENGTH = 32; + public static final int MIN_LOCKTIME = 1500000000; + + public static final long NO_LOCKTIME_NO_RBF_SEQUENCE = 0xFFFFFFFFL; + public static final long LOCKTIME_NO_RBF_SEQUENCE = NO_LOCKTIME_NO_RBF_SEQUENCE - 1; + + // Assuming node's trade-bot has no more than 100 entries? + private static final int MAX_CACHE_ENTRIES = 100; + + // Max time-to-live for cache entries (milliseconds) + private static final long CACHE_TIMEOUT = 30_000L; + + @SuppressWarnings("serial") + private static final Map SECRET_CACHE = new LinkedHashMap<>(MAX_CACHE_ENTRIES + 1, 0.75F, true) { + // This method is called just after a new entry has been added + @Override + public boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_CACHE_ENTRIES; + } + }; + private static final byte[] NO_SECRET_CACHE_ENTRY = new byte[0]; + + @SuppressWarnings("serial") + private static final Map STATUS_CACHE = new LinkedHashMap<>(MAX_CACHE_ENTRIES + 1, 0.75F, true) { + // This method is called just after a new entry has been added + @Override + public boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_CACHE_ENTRIES; + } + }; + + /* + * OP_TUCK (to copy public key to before signature) + * OP_CHECKSIGVERIFY (sig & pubkey must verify or script fails) + * OP_HASH160 (convert public key to PKH) + * OP_DUP (duplicate PKH) + * OP_EQUAL (does PKH match refund PKH?) + * OP_IF + * OP_DROP (no need for duplicate PKH) + * + * OP_CHECKLOCKTIMEVERIFY (if this passes, leftover stack is so script passes) + * OP_ELSE + * OP_EQUALVERIFY (duplicate PKH must match redeem PKH or script fails) + * OP_HASH160 (hash secret) + * OP_EQUAL (do hashes of secrets match? if true, script passes else script fails) + * OP_ENDIF + */ + + private static final byte[] redeemScript1 = HashCode.fromString("7dada97614").asBytes(); // OP_TUCK OP_CHECKSIGVERIFY OP_HASH160 OP_DUP push(0x14 bytes) + private static final byte[] redeemScript2 = HashCode.fromString("87637504").asBytes(); // OP_EQUAL OP_IF OP_DROP push(0x4 bytes) + private static final byte[] redeemScript3 = HashCode.fromString("b16714").asBytes(); // OP_CHECKLOCKTIMEVERIFY OP_ELSE push(0x14 bytes) + private static final byte[] redeemScript4 = HashCode.fromString("88a914").asBytes(); // OP_EQUALVERIFY OP_HASH160 push(0x14 bytes) + private static final byte[] redeemScript5 = HashCode.fromString("8768").asBytes(); // OP_EQUAL OP_ENDIF + + /** + * Returns redeemScript used for cross-chain trading. + *

+ * See comments in {@link BitcoinyHTLC} for more details. + * + * @param refunderPubKeyHash 20-byte HASH160 of P2SH funder's public key, for refunding purposes + * @param lockTime seconds-since-epoch threshold, after which P2SH funder can claim refund + * @param redeemerPubKeyHash 20-byte HASH160 of P2SH redeemer's public key + * @param hashOfSecret 20-byte HASH160 of secret, used by P2SH redeemer to claim funds + */ + public static byte[] buildScript(byte[] refunderPubKeyHash, int lockTime, byte[] redeemerPubKeyHash, byte[] hashOfSecret) { + return Bytes.concat(redeemScript1, refunderPubKeyHash, redeemScript2, BitTwiddling.toLEByteArray((int) (lockTime & 0xffffffffL)), + redeemScript3, redeemerPubKeyHash, redeemScript4, hashOfSecret, redeemScript5); + } + + /** + * Builds a custom transaction to spend HTLC P2SH. + * + * @param params blockchain network parameters + * @param amount output amount, should be total of input amounts, less miner fees + * @param spendKey key for signing transaction, and also where funds are 'sent' (output) + * @param fundingOutput output from transaction that funded P2SH address + * @param redeemScriptBytes the redeemScript itself, in byte[] form + * @param lockTime (optional) transaction nLockTime, used in refund scenario + * @param scriptSigBuilder function for building scriptSig using transaction input signature + * @param outputPublicKeyHash PKH used to create P2PKH output + * @return Signed transaction for spending P2SH + */ + public static Transaction buildP2shTransaction(NetworkParameters params, Coin amount, ECKey spendKey, + List fundingOutputs, byte[] redeemScriptBytes, + Long lockTime, Function scriptSigBuilder, byte[] outputPublicKeyHash) { + Transaction transaction = new Transaction(params); + transaction.setVersion(2); + + // Output is back to P2SH funder + transaction.addOutput(amount, ScriptBuilder.createP2PKHOutputScript(outputPublicKeyHash)); + + for (int inputIndex = 0; inputIndex < fundingOutputs.size(); ++inputIndex) { + TransactionOutput fundingOutput = fundingOutputs.get(inputIndex); + + // Input (without scriptSig prior to signing) + TransactionInput input = new TransactionInput(params, null, redeemScriptBytes, fundingOutput.getOutPointFor()); + if (lockTime != null) + input.setSequenceNumber(LOCKTIME_NO_RBF_SEQUENCE); // Use max-value - 1, so lockTime can be used but not RBF + else + input.setSequenceNumber(NO_LOCKTIME_NO_RBF_SEQUENCE); // Use max-value, so no lockTime and no RBF + transaction.addInput(input); + } + + // Set locktime after inputs added but before input signatures are generated + if (lockTime != null) + transaction.setLockTime(lockTime); + + for (int inputIndex = 0; inputIndex < fundingOutputs.size(); ++inputIndex) { + // Generate transaction signature for input + final boolean anyoneCanPay = false; + TransactionSignature txSig = transaction.calculateSignature(inputIndex, spendKey, redeemScriptBytes, SigHash.ALL, anyoneCanPay); + + // Calculate transaction signature + byte[] txSigBytes = txSig.encodeToBitcoin(); + + // Build scriptSig using lambda and tx signature + Script scriptSig = scriptSigBuilder.apply(txSigBytes); + + // Set input scriptSig + transaction.getInput(inputIndex).setScriptSig(scriptSig); + } + + return transaction; + } + + /** + * Returns signed transaction claiming refund from HTLC P2SH. + * + * @param params blockchain network parameters + * @param refundAmount refund amount, should be total of input amounts, less miner fees + * @param refundKey key for signing transaction + * @param fundingOutputs outputs from transaction that funded P2SH address + * @param redeemScriptBytes the redeemScript itself, in byte[] form + * @param lockTime transaction nLockTime - must be at least locktime used in redeemScript + * @param receivingAccountInfo public-key-hash used for P2PKH output + * @return Signed transaction for refunding P2SH + */ + public static Transaction buildRefundTransaction(NetworkParameters params, Coin refundAmount, ECKey refundKey, + List fundingOutputs, byte[] redeemScriptBytes, long lockTime, byte[] receivingAccountInfo) { + Function refundSigScriptBuilder = (txSigBytes) -> { + // Build scriptSig with... + ScriptBuilder scriptBuilder = new ScriptBuilder(); + + // transaction signature + scriptBuilder.addChunk(new ScriptChunk(txSigBytes.length, txSigBytes)); + + // redeem public key + byte[] refundPubKey = refundKey.getPubKey(); + scriptBuilder.addChunk(new ScriptChunk(refundPubKey.length, refundPubKey)); + + // redeem script + scriptBuilder.addChunk(new ScriptChunk(ScriptOpCodes.OP_PUSHDATA1, redeemScriptBytes)); + + return scriptBuilder.build(); + }; + + // Send funds back to funding address + return buildP2shTransaction(params, refundAmount, refundKey, fundingOutputs, redeemScriptBytes, lockTime, refundSigScriptBuilder, receivingAccountInfo); + } + + /** + * Returns signed transaction redeeming funds from P2SH address. + * + * @param params blockchain network parameters + * @param redeemAmount redeem amount, should be total of input amounts, less miner fees + * @param redeemKey key for signing transaction + * @param fundingOutputs outputs from transaction that funded P2SH address + * @param redeemScriptBytes the redeemScript itself, in byte[] form + * @param secret actual 32-byte secret used when building redeemScript + * @param receivingAccountInfo Bitcoin PKH used for output + * @return Signed transaction for redeeming P2SH + */ + public static Transaction buildRedeemTransaction(NetworkParameters params, Coin redeemAmount, ECKey redeemKey, + List fundingOutputs, byte[] redeemScriptBytes, byte[] secret, byte[] receivingAccountInfo) { + Function redeemSigScriptBuilder = (txSigBytes) -> { + // Build scriptSig with... + ScriptBuilder scriptBuilder = new ScriptBuilder(); + + // secret + scriptBuilder.addChunk(new ScriptChunk(secret.length, secret)); + + // transaction signature + scriptBuilder.addChunk(new ScriptChunk(txSigBytes.length, txSigBytes)); + + // redeem public key + byte[] redeemPubKey = redeemKey.getPubKey(); + scriptBuilder.addChunk(new ScriptChunk(redeemPubKey.length, redeemPubKey)); + + // redeem script + scriptBuilder.addChunk(new ScriptChunk(ScriptOpCodes.OP_PUSHDATA1, redeemScriptBytes)); + + return scriptBuilder.build(); + }; + + return buildP2shTransaction(params, redeemAmount, redeemKey, fundingOutputs, redeemScriptBytes, null, redeemSigScriptBuilder, receivingAccountInfo); + } + + /** + * Returns 'secret', if any, given HTLC's P2SH address. + *

+ * @throws ForeignBlockchainException + */ + public static byte[] findHtlcSecret(Bitcoiny bitcoiny, String p2shAddress) throws ForeignBlockchainException { + NetworkParameters params = bitcoiny.getNetworkParameters(); + String compoundKey = String.format("%s-%s-%d", params.getId(), p2shAddress, System.currentTimeMillis() / CACHE_TIMEOUT); + + byte[] secret = SECRET_CACHE.getOrDefault(compoundKey, NO_SECRET_CACHE_ENTRY); + if (secret != NO_SECRET_CACHE_ENTRY) + return secret; + + List rawTransactions = bitcoiny.getAddressTransactions(p2shAddress); + + for (byte[] rawTransaction : rawTransactions) { + Transaction transaction = new Transaction(params, rawTransaction); + + // Cycle through inputs, looking for one that spends our HTLC + for (TransactionInput input : transaction.getInputs()) { + Script scriptSig = input.getScriptSig(); + List scriptChunks = scriptSig.getChunks(); + + // Expected number of script chunks for redeem. Refund might not have the same number. + int expectedChunkCount = 1 /*secret*/ + 1 /*sig*/ + 1 /*pubkey*/ + 1 /*redeemScript*/; + if (scriptChunks.size() != expectedChunkCount) + continue; + + // We're expecting last chunk to contain the actual redeemScript + ScriptChunk lastChunk = scriptChunks.get(scriptChunks.size() - 1); + byte[] redeemScriptBytes = lastChunk.data; + + // If non-push scripts, redeemScript will be null + if (redeemScriptBytes == null) + continue; + + byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + Address inputAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); + + if (!inputAddress.toString().equals(p2shAddress)) + // Input isn't spending our HTLC + continue; + + secret = scriptChunks.get(0).data; + if (secret.length != BitcoinyHTLC.SECRET_LENGTH) + continue; + + // Cache secret for a while + SECRET_CACHE.put(compoundKey, secret); + + return secret; + } + } + + // Cache negative result + SECRET_CACHE.put(compoundKey, null); + + return null; + } + + /** + * Returns HTLC status, given P2SH address and expected redeem/refund amount + *

+ * @throws ForeignBlockchainException if error occurs + */ + public static Status determineHtlcStatus(BitcoinyBlockchainProvider blockchain, String p2shAddress, long minimumAmount) throws ForeignBlockchainException { + String compoundKey = String.format("%s-%s-%d", blockchain.getNetId(), p2shAddress, System.currentTimeMillis() / CACHE_TIMEOUT); + + Status cachedStatus = STATUS_CACHE.getOrDefault(compoundKey, null); + if (cachedStatus != null) + return cachedStatus; + + byte[] ourScriptPubKey = addressToScriptPubKey(p2shAddress); + List transactionHashes = blockchain.getAddressTransactions(ourScriptPubKey, BitcoinyBlockchainProvider.INCLUDE_UNCONFIRMED); + + // Sort by confirmed first, followed by ascending height + transactionHashes.sort(TransactionHash.CONFIRMED_FIRST.thenComparing(TransactionHash::getHeight)); + + // Transaction cache + Map transactionsByHash = new HashMap<>(); + // HASH160(redeem script) for this p2shAddress + byte[] ourRedeemScriptHash = addressToRedeemScriptHash(p2shAddress); + + // Check for spends first, caching full transaction info as we progress just in case we don't return in this loop + for (TransactionHash transactionInfo : transactionHashes) { + BitcoinyTransaction bitcoinyTransaction = blockchain.getTransaction(transactionInfo.txHash); + + // Cache for possible later reuse + transactionsByHash.put(transactionInfo.txHash, bitcoinyTransaction); + + // Acceptable funding is one transaction output, so we're expecting only one input + if (bitcoinyTransaction.inputs.size() != 1) + // Wrong number of inputs + continue; + + String scriptSig = bitcoinyTransaction.inputs.get(0).scriptSig; + + List scriptSigChunks = extractScriptSigChunks(HashCode.fromString(scriptSig).asBytes()); + if (scriptSigChunks.size() < 3 || scriptSigChunks.size() > 4) + // Not valid chunks for our form of HTLC + continue; + + // Last chunk is redeem script + byte[] redeemScriptBytes = scriptSigChunks.get(scriptSigChunks.size() - 1); + byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + if (!Arrays.equals(redeemScriptHash, ourRedeemScriptHash)) + // Not spending our specific HTLC redeem script + continue; + + if (scriptSigChunks.size() == 4) + // If we have 4 chunks, then secret is present, hence redeem + cachedStatus = transactionInfo.height == 0 ? Status.REDEEM_IN_PROGRESS : Status.REDEEMED; + else + cachedStatus = transactionInfo.height == 0 ? Status.REFUND_IN_PROGRESS : Status.REFUNDED; + + STATUS_CACHE.put(compoundKey, cachedStatus); + return cachedStatus; + } + + String ourScriptPubKeyHex = HashCode.fromBytes(ourScriptPubKey).toString(); + + // Check for funding + for (TransactionHash transactionInfo : transactionHashes) { + BitcoinyTransaction bitcoinyTransaction = transactionsByHash.get(transactionInfo.txHash); + if (bitcoinyTransaction == null) + // Should be present in map! + throw new ForeignBlockchainException("Cached Bitcoin transaction now missing?"); + + // Check outputs for our specific P2SH + for (BitcoinyTransaction.Output output : bitcoinyTransaction.outputs) { + // Check amount + if (output.value < minimumAmount) + // Output amount too small (not taking fees into account) + continue; + + String scriptPubKeyHex = output.scriptPubKey; + if (!scriptPubKeyHex.equals(ourScriptPubKeyHex)) + // Not funding our specific P2SH + continue; + + cachedStatus = transactionInfo.height == 0 ? Status.FUNDING_IN_PROGRESS : Status.FUNDED; + STATUS_CACHE.put(compoundKey, cachedStatus); + return cachedStatus; + } + } + + cachedStatus = Status.UNFUNDED; + STATUS_CACHE.put(compoundKey, cachedStatus); + return cachedStatus; + } + + private static List extractScriptSigChunks(byte[] scriptSigBytes) { + List chunks = new ArrayList<>(); + + int offset = 0; + int previousOffset = 0; + while (offset < scriptSigBytes.length) { + byte pushOp = scriptSigBytes[offset++]; + + if (pushOp < 0 || pushOp > 0x4c) + // Unacceptable OP + return Collections.emptyList(); + + // Special treatment for OP_PUSHDATA1 + if (pushOp == 0x4c) { + if (offset >= scriptSigBytes.length) + // Run out of scriptSig bytes? + return Collections.emptyList(); + + pushOp = scriptSigBytes[offset++]; + } + + previousOffset = offset; + offset += Byte.toUnsignedInt(pushOp); + + byte[] chunk = Arrays.copyOfRange(scriptSigBytes, previousOffset, offset); + chunks.add(chunk); + } + + return chunks; + } + + private static byte[] addressToScriptPubKey(String p2shAddress) { + // We want the HASH160 part of the P2SH address + byte[] p2shAddressBytes = Base58.decode(p2shAddress); + + byte[] scriptPubKey = new byte[1 + 1 + 20 + 1]; + scriptPubKey[0x00] = (byte) 0xa9; /* OP_HASH160 */ + scriptPubKey[0x01] = (byte) 0x14; /* PUSH 0x14 bytes */ + System.arraycopy(p2shAddressBytes, 1, scriptPubKey, 2, 0x14); + scriptPubKey[0x16] = (byte) 0x87; /* OP_EQUAL */ + + return scriptPubKey; + } + + private static byte[] addressToRedeemScriptHash(String p2shAddress) { + // We want the HASH160 part of the P2SH address + byte[] p2shAddressBytes = Base58.decode(p2shAddress); + + return Arrays.copyOfRange(p2shAddressBytes, 1, 1 + 20); + } + +} diff --git a/src/main/java/org/qortal/crosschain/BitcoinyTransaction.java b/src/main/java/org/qortal/crosschain/BitcoinyTransaction.java new file mode 100644 index 000000000..caf0b36de --- /dev/null +++ b/src/main/java/org/qortal/crosschain/BitcoinyTransaction.java @@ -0,0 +1,146 @@ +package org.qortal.crosschain; + +import java.util.List; +import java.util.stream.Collectors; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlTransient; + +@XmlAccessorType(XmlAccessType.FIELD) +public class BitcoinyTransaction { + + public final String txHash; + + @XmlTransient + public final int size; + + @XmlTransient + public final int locktime; + + // Not present if transaction is unconfirmed + public final Integer timestamp; + + public static class Input { + @XmlTransient + public final String scriptSig; + + @XmlTransient + public final int sequence; + + public final String outputTxHash; + + public final int outputVout; + + // For JAXB + protected Input() { + this.scriptSig = null; + this.sequence = 0; + this.outputTxHash = null; + this.outputVout = 0; + } + + public Input(String scriptSig, int sequence, String outputTxHash, int outputVout) { + this.scriptSig = scriptSig; + this.sequence = sequence; + this.outputTxHash = outputTxHash; + this.outputVout = outputVout; + } + + public String toString() { + return String.format("{output %s:%d, sequence %d, scriptSig %s}", + this.outputTxHash, this.outputVout, this.sequence, this.scriptSig); + } + } + @XmlTransient + public final List inputs; + + public static class Output { + @XmlTransient + public final String scriptPubKey; + + public final long value; + + public final List addresses; + + // For JAXB + protected Output() { + this.scriptPubKey = null; + this.value = 0; + this.addresses = null; + } + + public Output(String scriptPubKey, long value) { + this.scriptPubKey = scriptPubKey; + this.value = value; + this.addresses = null; + } + + public Output(String scriptPubKey, long value, List addresses) { + this.scriptPubKey = scriptPubKey; + this.value = value; + this.addresses = addresses; + } + + public String toString() { + return String.format("{value %d, scriptPubKey %s}", this.value, this.scriptPubKey); + } + } + public final List outputs; + + public final long totalAmount; + + // For JAXB + protected BitcoinyTransaction() { + this.txHash = null; + this.size = 0; + this.locktime = 0; + this.timestamp = 0; + this.inputs = null; + this.outputs = null; + this.totalAmount = 0; + } + + public BitcoinyTransaction(String txHash, int size, int locktime, Integer timestamp, + List inputs, List outputs) { + this.txHash = txHash; + this.size = size; + this.locktime = locktime; + this.timestamp = timestamp; + this.inputs = inputs; + this.outputs = outputs; + + this.totalAmount = outputs.stream().map(output -> output.value).reduce(0L, Long::sum); + } + + public String toString() { + return String.format("txHash %s, size %d, locktime %d, timestamp %d\n" + + "\tinputs: [%s]\n" + + "\toutputs: [%s]\n", + this.txHash, + this.size, + this.locktime, + this.timestamp, + this.inputs.stream().map(Input::toString).collect(Collectors.joining(",\n\t\t")), + this.outputs.stream().map(Output::toString).collect(Collectors.joining(",\n\t\t"))); + } + + @Override + public boolean equals(Object other) { + if (other == this) + return true; + + if (!(other instanceof BitcoinyTransaction)) + return false; + + BitcoinyTransaction otherTransaction = (BitcoinyTransaction) other; + + return this.txHash.equals(otherTransaction.txHash); + } + + @Override + public int hashCode() { + return this.txHash.hashCode(); + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/Dogecoin.java b/src/main/java/org/qortal/crosschain/Dogecoin.java new file mode 100644 index 000000000..6a70bb00d --- /dev/null +++ b/src/main/java/org/qortal/crosschain/Dogecoin.java @@ -0,0 +1,172 @@ +package org.qortal.crosschain; + +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.Context; +import org.bitcoinj.core.NetworkParameters; +import org.libdohj.params.DogecoinMainNetParams; +//import org.libdohj.params.DogecoinRegTestParams; +import org.libdohj.params.DogecoinTestNet3Params; +import org.qortal.crosschain.ElectrumX.Server; +import org.qortal.crosschain.ElectrumX.Server.ConnectionType; +import org.qortal.settings.Settings; + +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumMap; +import java.util.Map; + +public class Dogecoin extends Bitcoiny { + + public static final String CURRENCY_CODE = "DOGE"; + + private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(1000000); // 0.01 DOGE per 1000 bytes + + private static final long MINIMUM_ORDER_AMOUNT = 100000000L; // 1 DOGE minimum order. See recommendations: + // https://github.com/dogecoin/dogecoin/blob/master/doc/fee-recommendation.md + + // Temporary values until a dynamic fee system is written. + private static final long MAINNET_FEE = 100000L; + private static final long NON_MAINNET_FEE = 10000L; // TODO: calibrate this + + private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ConnectionType.class); + static { + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.TCP, 50001); + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.SSL, 50002); + } + + public enum DogecoinNet { + MAIN { + @Override + public NetworkParameters getParams() { + return DogecoinMainNetParams.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + // Servers chosen on NO BASIS WHATSOEVER from various sources! + new Server("electrum1.cipig.net", ConnectionType.TCP, 10060), + new Server("electrum2.cipig.net", ConnectionType.TCP, 10060), + new Server("electrum3.cipig.net", ConnectionType.TCP, 10060)); + // TODO: add more mainnet servers. It's too centralized. + } + + @Override + public String getGenesisHash() { + return "1a91e3dace36e2be3bf030a65679fe821aa1d6ef92e7c9902eb318182c355691"; + } + + @Override + public long getP2shFee(Long timestamp) { + // TODO: This will need to be replaced with something better in the near future! + return MAINNET_FEE; + } + }, + TEST3 { + @Override + public NetworkParameters getParams() { + return DogecoinTestNet3Params.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList(); // TODO: find testnet servers + } + + @Override + public String getGenesisHash() { + return "4966625a4b2851d9fdee139e56211a0d88575f59ed816ff5e6a63deb4e3e29a0"; + } + + @Override + public long getP2shFee(Long timestamp) { + return NON_MAINNET_FEE; + } + }, + REGTEST { + @Override + public NetworkParameters getParams() { + return null; // TODO: DogecoinRegTestParams.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + new Server("localhost", ConnectionType.TCP, 50001), + new Server("localhost", ConnectionType.SSL, 50002)); + } + + @Override + public String getGenesisHash() { + // This is unique to each regtest instance + return null; + } + + @Override + public long getP2shFee(Long timestamp) { + return NON_MAINNET_FEE; + } + }; + + public abstract NetworkParameters getParams(); + public abstract Collection getServers(); + public abstract String getGenesisHash(); + public abstract long getP2shFee(Long timestamp) throws ForeignBlockchainException; + } + + private static Dogecoin instance; + + private final DogecoinNet dogecoinNet; + + // Constructors and instance + + private Dogecoin(DogecoinNet dogecoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { + super(blockchain, bitcoinjContext, currencyCode); + this.dogecoinNet = dogecoinNet; + + LOGGER.info(() -> String.format("Starting Dogecoin support using %s", this.dogecoinNet.name())); + } + + public static synchronized Dogecoin getInstance() { + if (instance == null) { + DogecoinNet dogecoinNet = Settings.getInstance().getDogecoinNet(); + + BitcoinyBlockchainProvider electrumX = new ElectrumX("Dogecoin-" + dogecoinNet.name(), dogecoinNet.getGenesisHash(), dogecoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS); + Context bitcoinjContext = new Context(dogecoinNet.getParams()); + + instance = new Dogecoin(dogecoinNet, electrumX, bitcoinjContext, CURRENCY_CODE); + } + + return instance; + } + + // Getters & setters + + public static synchronized void resetForTesting() { + instance = null; + } + + // Actual useful methods for use by other classes + + @Override + public Coin getFeePerKb() { + return DEFAULT_FEE_PER_KB; + } + + @Override + public long getMinimumOrderAmount() { + return MINIMUM_ORDER_AMOUNT; + } + + /** + * Returns estimated DOGE fee, in sats per 1000bytes, optionally for historic timestamp. + * + * @param timestamp optional milliseconds since epoch, or null for 'now' + * @return sats per 1000bytes, or throws ForeignBlockchainException if something went wrong + */ + @Override + public long getP2shFee(Long timestamp) throws ForeignBlockchainException { + return this.dogecoinNet.getP2shFee(timestamp); + } + +} diff --git a/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java b/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java new file mode 100644 index 000000000..36ff7c5c4 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java @@ -0,0 +1,855 @@ +package org.qortal.crosschain; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ciyam.at.*; +import org.qortal.account.Account; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.utils.Base58; +import org.qortal.utils.BitTwiddling; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import static org.ciyam.at.OpCode.calcOffset; + +/** + * Cross-chain trade AT + * + *

+ *

    + *
  • Bob generates Dogecoin & Qortal 'trade' keys + *
      + *
    • private key required to sign P2SH redeem tx
      • + *
    • private key could be used to create 'secret' (e.g. double-SHA256)
      • + *
    • encrypted private key could be stored in Qortal AT for access by Bob from any node
      • + *
    + *
    • + *
  • Bob deploys Qortal AT + *
      + *
    + *
    • + *
  • Alice finds Qortal AT and wants to trade + *
      + *
    • Alice generates Dogecoin & Qortal 'trade' keys
      • + *
    • Alice funds Dogecoin P2SH-A
      • + *
    • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: + *
        + *
      • hash-of-secret-A
        • + *
      • her 'trade' Dogecoin PKH
        • + *
      + *
      • + *
    + *
    • + *
  • Bob receives "offer" MESSAGE + *
      + *
    • Checks Alice's P2SH-A
      • + *
    • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: + *
        + *
      • Alice's trade Qortal address
        • + *
      • Alice's trade Dogecoin PKH
        • + *
      • hash-of-secret-A
        • + *
      + *
      • + *
    + *
    • + *
  • Alice checks Qortal AT to confirm it's locked to her + *
      + *
    • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: + *
        + *
      • secret-A
        • + *
      • Qortal receiving address of her chosing
        • + *
      + *
      • + *
    • AT's QORT funds are sent to Qortal receiving address
      • + *
    + *
    • + *
  • Bob checks AT, extracts secret-A + *
      + *
    • Bob redeems P2SH-A using his Dogecoin trade key and secret-A
      • + *
    • P2SH-A DOGE funds end up at Dogecoin address determined by redeem transaction output(s)
      • + *
    + *
    • + *
+ */ +public class DogecoinACCTv1 implements ACCT { + + private static final Logger LOGGER = LogManager.getLogger(DogecoinACCTv1.class); + + public static final String NAME = DogecoinACCTv1.class.getSimpleName(); + public static final byte[] CODE_BYTES_HASH = HashCode.fromString("0eb49b0313ff3855a29d860c2a8203faa2ef62e28ea30459321f176079cfa3a5").asBytes(); // SHA256 of AT code bytes + + public static final int SECRET_LENGTH = 32; + + /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ + private static final int MODE_VALUE_OFFSET = 61; + /** Byte offset into AT state data where 'mode' variable (long) is stored. */ + public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); + + public static class OfferMessageData { + public byte[] partnerDogecoinPKH; + public byte[] hashOfSecretA; + public long lockTimeA; + } + public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerDogecoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; + public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ + + 24 /*partner's Dogecoin PKH (padded from 20 to 24)*/ + + 8 /*AT trade timeout (minutes)*/ + + 24 /*hash of secret-A (padded from 20 to 24)*/ + + 8 /*lockTimeA*/; + public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret-A*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; + public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; + + private static DogecoinACCTv1 instance; + + private DogecoinACCTv1() { + } + + public static synchronized DogecoinACCTv1 getInstance() { + if (instance == null) + instance = new DogecoinACCTv1(); + + return instance; + } + + @Override + public byte[] getCodeBytesHash() { + return CODE_BYTES_HASH; + } + + @Override + public int getModeByteOffset() { + return MODE_BYTE_OFFSET; + } + + @Override + public ForeignBlockchain getBlockchain() { + return Dogecoin.getInstance(); + } + + /** + * Returns Qortal AT creation bytes for cross-chain trading AT. + *

+ * tradeTimeout (minutes) is the time window for the trade partner to send the + * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. + * + * @param creatorTradeAddress AT creator's trade Qortal address + * @param dogecoinPublicKeyHash 20-byte HASH160 of creator's trade Dogecoin public key + * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT + * @param dogecoinAmount how much DOGE the AT creator is expecting to trade + * @param tradeTimeout suggested timeout for entire trade + */ + public static byte[] buildQortalAT(String creatorTradeAddress, byte[] dogecoinPublicKeyHash, long qortAmount, long dogecoinAmount, int tradeTimeout) { + if (dogecoinPublicKeyHash.length != 20) + throw new IllegalArgumentException("Dogecoin public key hash should be 20 bytes"); + + // Labels for data segment addresses + int addrCounter = 0; + + // Constants (with corresponding dataByteBuffer.put*() calls below) + + final int addrCreatorTradeAddress1 = addrCounter++; + final int addrCreatorTradeAddress2 = addrCounter++; + final int addrCreatorTradeAddress3 = addrCounter++; + final int addrCreatorTradeAddress4 = addrCounter++; + + final int addrDogecoinPublicKeyHash = addrCounter; + addrCounter += 4; + + final int addrQortAmount = addrCounter++; + final int addrDogecoinAmount = addrCounter++; + final int addrTradeTimeout = addrCounter++; + + final int addrMessageTxnType = addrCounter++; + final int addrExpectedTradeMessageLength = addrCounter++; + final int addrExpectedRedeemMessageLength = addrCounter++; + + final int addrCreatorAddressPointer = addrCounter++; + final int addrQortalPartnerAddressPointer = addrCounter++; + final int addrMessageSenderPointer = addrCounter++; + + final int addrTradeMessagePartnerDogecoinPKHOffset = addrCounter++; + final int addrPartnerDogecoinPKHPointer = addrCounter++; + final int addrTradeMessageHashOfSecretAOffset = addrCounter++; + final int addrHashOfSecretAPointer = addrCounter++; + + final int addrRedeemMessageReceivingAddressOffset = addrCounter++; + + final int addrMessageDataPointer = addrCounter++; + final int addrMessageDataLength = addrCounter++; + + final int addrPartnerReceivingAddressPointer = addrCounter++; + + final int addrEndOfConstants = addrCounter; + + // Variables + + final int addrCreatorAddress1 = addrCounter++; + final int addrCreatorAddress2 = addrCounter++; + final int addrCreatorAddress3 = addrCounter++; + final int addrCreatorAddress4 = addrCounter++; + + final int addrQortalPartnerAddress1 = addrCounter++; + final int addrQortalPartnerAddress2 = addrCounter++; + final int addrQortalPartnerAddress3 = addrCounter++; + final int addrQortalPartnerAddress4 = addrCounter++; + + final int addrLockTimeA = addrCounter++; + final int addrRefundTimeout = addrCounter++; + final int addrRefundTimestamp = addrCounter++; + final int addrLastTxnTimestamp = addrCounter++; + final int addrBlockTimestamp = addrCounter++; + final int addrTxnType = addrCounter++; + final int addrResult = addrCounter++; + + final int addrMessageSender1 = addrCounter++; + final int addrMessageSender2 = addrCounter++; + final int addrMessageSender3 = addrCounter++; + final int addrMessageSender4 = addrCounter++; + + final int addrMessageLength = addrCounter++; + + final int addrMessageData = addrCounter; + addrCounter += 4; + + final int addrHashOfSecretA = addrCounter; + addrCounter += 4; + + final int addrPartnerDogecoinPKH = addrCounter; + addrCounter += 4; + + final int addrPartnerReceivingAddress = addrCounter; + addrCounter += 4; + + final int addrMode = addrCounter++; + assert addrMode == MODE_VALUE_OFFSET : String.format("addrMode %d does not match MODE_VALUE_OFFSET %d", addrMode, MODE_VALUE_OFFSET); + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // AT creator's trade Qortal address, decoded from Base58 + assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; + byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); + dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); + + // Dogecoin public key hash + assert dataByteBuffer.position() == addrDogecoinPublicKeyHash * MachineState.VALUE_SIZE : "addrDogecoinPublicKeyHash incorrect"; + dataByteBuffer.put(Bytes.ensureCapacity(dogecoinPublicKeyHash, 32, 0)); + + // Redeem Qort amount + assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; + dataByteBuffer.putLong(qortAmount); + + // Expected Dogecoin amount + assert dataByteBuffer.position() == addrDogecoinAmount * MachineState.VALUE_SIZE : "addrDogecoinAmount incorrect"; + dataByteBuffer.putLong(dogecoinAmount); + + // Suggested trade timeout (minutes) + assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; + dataByteBuffer.putLong(tradeTimeout); + + // We're only interested in MESSAGE transactions + assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; + dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); + + // Expected length of 'trade' MESSAGE data from AT creator + assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; + dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); + + // Expected length of 'redeem' MESSAGE data from trade partner + assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; + dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); + + // Index into data segment of AT creator's address, used by GET_B_IND + assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; + dataByteBuffer.putLong(addrCreatorAddress1); + + // Index into data segment of partner's Qortal address, used by SET_B_IND + assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; + dataByteBuffer.putLong(addrQortalPartnerAddress1); + + // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND + assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; + dataByteBuffer.putLong(addrMessageSender1); + + // Offset into 'trade' MESSAGE data payload for extracting partner's Dogecoin PKH + assert dataByteBuffer.position() == addrTradeMessagePartnerDogecoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerDogecoinPKHOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Index into data segment of partner's Dogecoin PKH, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerDogecoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerDogecoinPKHPointer incorrect"; + dataByteBuffer.putLong(addrPartnerDogecoinPKH); + + // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A + assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; + dataByteBuffer.putLong(64L); + + // Index into data segment to hash of secret A, used by GET_B_IND + assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; + dataByteBuffer.putLong(addrHashOfSecretA); + + // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address + assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Source location and length for hashing any passed secret + assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; + dataByteBuffer.putLong(addrMessageData); + assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; + dataByteBuffer.putLong(32L); + + // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; + dataByteBuffer.putLong(addrPartnerReceivingAddress); + + assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; + + // Code labels + Integer labelRefund = null; + + Integer labelTradeTxnLoop = null; + Integer labelCheckTradeTxn = null; + Integer labelCheckCancelTxn = null; + Integer labelNotTradeNorCancelTxn = null; + Integer labelCheckNonRefundTradeTxn = null; + Integer labelTradeTxnExtract = null; + Integer labelRedeemTxnLoop = null; + Integer labelCheckRedeemTxn = null; + Integer labelCheckRedeemTxnSender = null; + Integer labelPayout = null; + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + /* NOP - to ensure DOGECOIN ACCT is unique */ + codeByteBuffer.put(OpCode.NOP.compile()); + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); + + // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ + + /* Transaction processing loop */ + labelTradeTxnLoop = codeByteBuffer.position(); + + // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckTradeTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); + + /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + // Message sender's address matches AT creator's trade address so go process 'trade' message + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); + + /* Checking message sender for possible cancel message */ + labelCheckCancelTxn = codeByteBuffer.position(); + + // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + // Partner address is AT creator's address, so cancel offer and finish. + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + /* Not trade nor cancel message */ + labelNotTradeNorCancelTxn = codeByteBuffer.position(); + + // Loop to find another transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Possible switch-to-trade-mode message */ + labelCheckNonRefundTradeTxn = codeByteBuffer.position(); + + // Check 'trade' message we received has expected number of message bytes + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to info extraction code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); + // Message length didn't match - go back to finding another 'trade' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Extracting info from 'trade' MESSAGE transaction */ + labelTradeTxnExtract = codeByteBuffer.position(); + + // Extract message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); + + // Extract trade partner's Dogecoin public key hash (PKH) from message into B + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerDogecoinPKHOffset)); + // Store partner's Dogecoin PKH (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerDogecoinPKHPointer)); + // Extract AT trade timeout (minutes) (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrRefundTimeout)); + + // Grab next 32 bytes + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); + + // Extract hash-of-secret-A (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); + // Extract lockTime-A (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); + + // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); + + /* We are in 'trade mode' */ + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ + + // Fetch current block 'timestamp' + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); + // If we're not past refund 'timestamp' then look for next transaction + codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + // We're past refund 'timestamp' so go refund everything back to AT creator + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); + + /* Transaction processing loop */ + labelRedeemTxnLoop = codeByteBuffer.position(); + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckRedeemTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check message payload length */ + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to sender checking code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); + // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Check transaction's sender */ + labelCheckRedeemTxnSender = codeByteBuffer.position(); + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check 'secret-A' in transaction's message */ + + // Extract secret-A from first 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); + // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); + // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). + // Save the equality result (1 if they match, 0 otherwise) into addrResult. + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); + // If hashes don't match, addrResult will be zero so go find another transaction + codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Success! Pay arranged amount to receiving address */ + labelPayout = codeByteBuffer.position(); + + // Extract Qortal receiving address from next 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); + // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); + // Pay AT's balance to receiving address + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); + // Set redeemed mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + // Fall-through to refunding any remaining balance back to AT creator + + /* Refund balance back to AT creator */ + labelRefund = codeByteBuffer.position(); + + // Set refunded mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile DOGE-QORT ACCT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + assert Arrays.equals(Crypto.digest(codeBytes), DogecoinACCTv1.CODE_BYTES_HASH) + : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { + ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + byte[] addressBytes = new byte[25]; // for general use + String atAddress = atStateData.getATAddress(); + + CrossChainTradeData tradeData = new CrossChainTradeData(); + + tradeData.foreignBlockchain = SupportedBlockchain.DOGECOIN.name(); + tradeData.acctName = NAME; + + tradeData.qortalAtAddress = atAddress; + tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); + tradeData.creationTimestamp = creationTimestamp; + + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + + byte[] stateData = atStateData.getStateData(); + ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); + dataByteBuffer.position(MachineState.HEADER_LENGTH); + + /* Constants */ + + // Skip creator's trade address + dataByteBuffer.get(addressBytes); + tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Creator's Dogecoin/foreign public key hash + tradeData.creatorForeignPKH = new byte[20]; + dataByteBuffer.get(tradeData.creatorForeignPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes + + // We don't use secret-B + tradeData.hashOfSecretB = null; + + // Redeem payout + tradeData.qortAmount = dataByteBuffer.getLong(); + + // Expected DOGE amount + tradeData.expectedForeignAmount = dataByteBuffer.getLong(); + + // Trade timeout + tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); + + // Skip MESSAGE transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'trade' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'redeem' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Qortal trade address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message sender + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for partner's Dogecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Dogecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'redeem' message data offset for partner's Qortal receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message data + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip message data length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + /* End of constants / begin variables */ + + // Skip AT creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Partner's trade address (if present) + dataByteBuffer.get(addressBytes); + String qortalRecipient = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Potential lockTimeA (if in trade mode) + int lockTimeA = (int) dataByteBuffer.getLong(); + + // AT refund timeout (probably only useful for debugging) + int refundTimeout = (int) dataByteBuffer.getLong(); + + // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) + long tradeRefundTimestamp = dataByteBuffer.getLong(); + + // Skip last transaction timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip block timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary result + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message sender + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Skip message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message data + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Potential hash160 of secret A + byte[] hashOfSecretA = new byte[20]; + dataByteBuffer.get(hashOfSecretA); + dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes + + // Potential partner's Dogecoin PKH + byte[] partnerDogecoinPKH = new byte[20]; + dataByteBuffer.get(partnerDogecoinPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerDogecoinPKH.length); // skip to 32 bytes + + // Partner's receiving address (if present) + byte[] partnerReceivingAddress = new byte[25]; + dataByteBuffer.get(partnerReceivingAddress); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes + + // Trade AT's 'mode' + long modeValue = dataByteBuffer.getLong(); + AcctMode mode = AcctMode.valueOf((int) (modeValue & 0xffL)); + + /* End of variables */ + + if (mode != null && mode != AcctMode.OFFERING) { + tradeData.mode = mode; + tradeData.refundTimeout = refundTimeout; + tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; + tradeData.qortalPartnerAddress = qortalRecipient; + tradeData.hashOfSecretA = hashOfSecretA; + tradeData.partnerForeignPKH = partnerDogecoinPKH; + tradeData.lockTimeA = lockTimeA; + + if (mode == AcctMode.REDEEMED) + tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); + } else { + tradeData.mode = AcctMode.OFFERING; + } + + tradeData.duplicateDeprecated(); + + return tradeData; + } + + /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ + public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); + } + + /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ + public static OfferMessageData extractOfferMessageData(byte[] messageData) { + if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) + return null; + + OfferMessageData offerMessageData = new OfferMessageData(); + offerMessageData.partnerDogecoinPKH = Arrays.copyOfRange(messageData, 0, 20); + offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); + offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); + + return offerMessageData; + } + + /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ + public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int refundTimeout) { + byte[] data = new byte[TRADE_MESSAGE_LENGTH]; + byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + byte[] refundTimeoutBytes = BitTwiddling.toBEByteArray((long) refundTimeout); + + System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); + System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); + System.arraycopy(refundTimeoutBytes, 0, data, 56, refundTimeoutBytes.length); + System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); + System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); + + return data; + } + + /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ + @Override + public byte[] buildCancelMessage(String creatorQortalAddress) { + byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; + byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); + + System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); + + return data; + } + + /** Returns 'redeem' MESSAGE payload for trade partner to send to AT. */ + public static byte[] buildRedeemMessage(byte[] secretA, String qortalReceivingAddress) { + byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; + byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); + + System.arraycopy(secretA, 0, data, 0, secretA.length); + System.arraycopy(qortalReceivingAddressBytes, 0, data, 32, qortalReceivingAddressBytes.length); + + return data; + } + + /** Returns refund timeout (minutes) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ + public static int calcRefundTimeout(long offerMessageTimestamp, int lockTimeA) { + // refund should be triggered halfway between offerMessageTimestamp and lockTimeA + return (int) ((lockTimeA - (offerMessageTimestamp / 1000L)) / 2L / 60L); + } + + @Override + public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { + String atAddress = crossChainTradeData.qortalAtAddress; + String redeemerAddress = crossChainTradeData.qortalPartnerAddress; + + // We don't have partner's public key so we check every message to AT + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); + if (messageTransactionsData == null) + return null; + + // Find 'redeem' message + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + // Check message payload type/encryption + if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) + continue; + + // Check message payload size + byte[] messageData = messageTransactionData.getData(); + if (messageData.length != REDEEM_MESSAGE_LENGTH) + // Wrong payload length + continue; + + // Check sender + if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) + // Wrong sender; + continue; + + // Extract secretA + byte[] secretA = new byte[32]; + System.arraycopy(messageData, 0, secretA, 0, secretA.length); + + byte[] hashOfSecretA = Crypto.hash160(secretA); + if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) + continue; + + return secretA; + } + + return null; + } + +} diff --git a/src/main/java/org/qortal/crosschain/DogecoinACCTv2.java b/src/main/java/org/qortal/crosschain/DogecoinACCTv2.java new file mode 100644 index 000000000..c4b0edb3d --- /dev/null +++ b/src/main/java/org/qortal/crosschain/DogecoinACCTv2.java @@ -0,0 +1,861 @@ +package org.qortal.crosschain; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ciyam.at.*; +import org.qortal.account.Account; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.utils.Base58; +import org.qortal.utils.BitTwiddling; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import static org.ciyam.at.OpCode.calcOffset; + +/** + * Cross-chain trade AT + * + *

+ *

    + *
  • Bob generates Dogecoin & Qortal 'trade' keys + *
      + *
    • private key required to sign P2SH redeem tx
      • + *
    • private key could be used to create 'secret' (e.g. double-SHA256)
      • + *
    • encrypted private key could be stored in Qortal AT for access by Bob from any node
      • + *
    + *
    • + *
  • Bob deploys Qortal AT + *
      + *
    + *
    • + *
  • Alice finds Qortal AT and wants to trade + *
      + *
    • Alice generates Dogecoin & Qortal 'trade' keys
      • + *
    • Alice funds Dogecoin P2SH-A
      • + *
    • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: + *
        + *
      • hash-of-secret-A
        • + *
      • her 'trade' Dogecoin PKH
        • + *
      + *
      • + *
    + *
    • + *
  • Bob receives "offer" MESSAGE + *
      + *
    • Checks Alice's P2SH-A
      • + *
    • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: + *
        + *
      • Alice's trade Qortal address
        • + *
      • Alice's trade Dogecoin PKH
        • + *
      • hash-of-secret-A
        • + *
      + *
      • + *
    + *
    • + *
  • Alice checks Qortal AT to confirm it's locked to her + *
      + *
    • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: + *
        + *
      • secret-A
        • + *
      • Qortal receiving address of her chosing
        • + *
      + *
      • + *
    • AT's QORT funds are sent to Qortal receiving address
      • + *
    + *
    • + *
  • Bob checks AT, extracts secret-A + *
      + *
    • Bob redeems P2SH-A using his Dogecoin trade key and secret-A
      • + *
    • P2SH-A DOGE funds end up at Dogecoin address determined by redeem transaction output(s)
      • + *
    + *
    • + *
+ */ +public class DogecoinACCTv2 implements ACCT { + + private static final Logger LOGGER = LogManager.getLogger(DogecoinACCTv2.class); + + public static final String NAME = DogecoinACCTv2.class.getSimpleName(); + public static final byte[] CODE_BYTES_HASH = HashCode.fromString("6fff38d6eeb06568a9c879c5628527730319844aa0de53f5f4ffab5506efe885").asBytes(); // SHA256 of AT code bytes + + public static final int SECRET_LENGTH = 32; + + /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ + private static final int MODE_VALUE_OFFSET = 61; + /** Byte offset into AT state data where 'mode' variable (long) is stored. */ + public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); + + public static class OfferMessageData { + public byte[] partnerDogecoinPKH; + public byte[] hashOfSecretA; + public long lockTimeA; + } + public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerDogecoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; + public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ + + 24 /*partner's Dogecoin PKH (padded from 20 to 24)*/ + + 8 /*AT trade timeout (minutes)*/ + + 24 /*hash of secret-A (padded from 20 to 24)*/ + + 8 /*lockTimeA*/; + public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret-A*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; + public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; + + private static DogecoinACCTv2 instance; + + private DogecoinACCTv2() { + } + + public static synchronized DogecoinACCTv2 getInstance() { + if (instance == null) + instance = new DogecoinACCTv2(); + + return instance; + } + + @Override + public byte[] getCodeBytesHash() { + return CODE_BYTES_HASH; + } + + @Override + public int getModeByteOffset() { + return MODE_BYTE_OFFSET; + } + + @Override + public ForeignBlockchain getBlockchain() { + return Dogecoin.getInstance(); + } + + /** + * Returns Qortal AT creation bytes for cross-chain trading AT. + *

+ * tradeTimeout (minutes) is the time window for the trade partner to send the + * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. + * + * @param creatorTradeAddress AT creator's trade Qortal address + * @param dogecoinPublicKeyHash 20-byte HASH160 of creator's trade Dogecoin public key + * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT + * @param dogecoinAmount how much DOGE the AT creator is expecting to trade + * @param tradeTimeout suggested timeout for entire trade + */ + public static byte[] buildQortalAT(String creatorTradeAddress, byte[] dogecoinPublicKeyHash, long qortAmount, long dogecoinAmount, int tradeTimeout) { + if (dogecoinPublicKeyHash.length != 20) + throw new IllegalArgumentException("Dogecoin public key hash should be 20 bytes"); + + // Labels for data segment addresses + int addrCounter = 0; + + // Constants (with corresponding dataByteBuffer.put*() calls below) + + final int addrCreatorTradeAddress1 = addrCounter++; + final int addrCreatorTradeAddress2 = addrCounter++; + final int addrCreatorTradeAddress3 = addrCounter++; + final int addrCreatorTradeAddress4 = addrCounter++; + + final int addrDogecoinPublicKeyHash = addrCounter; + addrCounter += 4; + + final int addrQortAmount = addrCounter++; + final int addrDogecoinAmount = addrCounter++; + final int addrTradeTimeout = addrCounter++; + + final int addrMessageTxnType = addrCounter++; + final int addrExpectedTradeMessageLength = addrCounter++; + final int addrExpectedRedeemMessageLength = addrCounter++; + + final int addrCreatorAddressPointer = addrCounter++; + final int addrQortalPartnerAddressPointer = addrCounter++; + final int addrMessageSenderPointer = addrCounter++; + + final int addrTradeMessagePartnerDogecoinPKHOffset = addrCounter++; + final int addrPartnerDogecoinPKHPointer = addrCounter++; + final int addrTradeMessageHashOfSecretAOffset = addrCounter++; + final int addrHashOfSecretAPointer = addrCounter++; + + final int addrRedeemMessageReceivingAddressOffset = addrCounter++; + + final int addrMessageDataPointer = addrCounter++; + final int addrMessageDataLength = addrCounter++; + + final int addrPartnerReceivingAddressPointer = addrCounter++; + + final int addrEndOfConstants = addrCounter; + + // Variables + + final int addrCreatorAddress1 = addrCounter++; + final int addrCreatorAddress2 = addrCounter++; + final int addrCreatorAddress3 = addrCounter++; + final int addrCreatorAddress4 = addrCounter++; + + final int addrQortalPartnerAddress1 = addrCounter++; + final int addrQortalPartnerAddress2 = addrCounter++; + final int addrQortalPartnerAddress3 = addrCounter++; + final int addrQortalPartnerAddress4 = addrCounter++; + + final int addrLockTimeA = addrCounter++; + final int addrRefundTimeout = addrCounter++; + final int addrRefundTimestamp = addrCounter++; + final int addrLastTxnTimestamp = addrCounter++; + final int addrBlockTimestamp = addrCounter++; + final int addrTxnType = addrCounter++; + final int addrResult = addrCounter++; + + final int addrMessageSender1 = addrCounter++; + final int addrMessageSender2 = addrCounter++; + final int addrMessageSender3 = addrCounter++; + final int addrMessageSender4 = addrCounter++; + + final int addrMessageLength = addrCounter++; + + final int addrMessageData = addrCounter; + addrCounter += 4; + + final int addrHashOfSecretA = addrCounter; + addrCounter += 4; + + final int addrPartnerDogecoinPKH = addrCounter; + addrCounter += 4; + + final int addrPartnerReceivingAddress = addrCounter; + addrCounter += 4; + + final int addrMode = addrCounter++; + assert addrMode == MODE_VALUE_OFFSET : String.format("addrMode %d does not match MODE_VALUE_OFFSET %d", addrMode, MODE_VALUE_OFFSET); + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // AT creator's trade Qortal address, decoded from Base58 + assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; + byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); + dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); + + // Dogecoin public key hash + assert dataByteBuffer.position() == addrDogecoinPublicKeyHash * MachineState.VALUE_SIZE : "addrDogecoinPublicKeyHash incorrect"; + dataByteBuffer.put(Bytes.ensureCapacity(dogecoinPublicKeyHash, 32, 0)); + + // Redeem Qort amount + assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; + dataByteBuffer.putLong(qortAmount); + + // Expected Dogecoin amount + assert dataByteBuffer.position() == addrDogecoinAmount * MachineState.VALUE_SIZE : "addrDogecoinAmount incorrect"; + dataByteBuffer.putLong(dogecoinAmount); + + // Suggested trade timeout (minutes) + assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; + dataByteBuffer.putLong(tradeTimeout); + + // We're only interested in MESSAGE transactions + assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; + dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); + + // Expected length of 'trade' MESSAGE data from AT creator + assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; + dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); + + // Expected length of 'redeem' MESSAGE data from trade partner + assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; + dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); + + // Index into data segment of AT creator's address, used by GET_B_IND + assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; + dataByteBuffer.putLong(addrCreatorAddress1); + + // Index into data segment of partner's Qortal address, used by SET_B_IND + assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; + dataByteBuffer.putLong(addrQortalPartnerAddress1); + + // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND + assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; + dataByteBuffer.putLong(addrMessageSender1); + + // Offset into 'trade' MESSAGE data payload for extracting partner's Dogecoin PKH + assert dataByteBuffer.position() == addrTradeMessagePartnerDogecoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerDogecoinPKHOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Index into data segment of partner's Dogecoin PKH, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerDogecoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerDogecoinPKHPointer incorrect"; + dataByteBuffer.putLong(addrPartnerDogecoinPKH); + + // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A + assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; + dataByteBuffer.putLong(64L); + + // Index into data segment to hash of secret A, used by GET_B_IND + assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; + dataByteBuffer.putLong(addrHashOfSecretA); + + // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address + assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Source location and length for hashing any passed secret + assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; + dataByteBuffer.putLong(addrMessageData); + assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; + dataByteBuffer.putLong(32L); + + // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; + dataByteBuffer.putLong(addrPartnerReceivingAddress); + + assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; + + // Code labels + Integer labelRefund = null; + + Integer labelTradeTxnLoop = null; + Integer labelCheckTradeTxn = null; + Integer labelCheckCancelTxn = null; + Integer labelNotTradeNorCancelTxn = null; + Integer labelCheckNonRefundTradeTxn = null; + Integer labelTradeTxnExtract = null; + Integer labelRedeemTxnLoop = null; + Integer labelCheckRedeemTxn = null; + Integer labelCheckRedeemTxnSender = null; + Integer labelPayout = null; + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + /* NOP - to ensure DOGECOIN ACCT is unique */ + codeByteBuffer.put(OpCode.NOP.compile()); + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); + + // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ + + /* Transaction processing loop */ + labelTradeTxnLoop = codeByteBuffer.position(); + + /* Sleep until message arrives */ + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.SLEEP_UNTIL_MESSAGE.value, addrLastTxnTimestamp)); + + // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckTradeTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); + + /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + // Message sender's address matches AT creator's trade address so go process 'trade' message + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); + + /* Checking message sender for possible cancel message */ + labelCheckCancelTxn = codeByteBuffer.position(); + + // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + // Partner address is AT creator's address, so cancel offer and finish. + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + /* Not trade nor cancel message */ + labelNotTradeNorCancelTxn = codeByteBuffer.position(); + + // Loop to find another transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Possible switch-to-trade-mode message */ + labelCheckNonRefundTradeTxn = codeByteBuffer.position(); + + // Check 'trade' message we received has expected number of message bytes + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to info extraction code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); + // Message length didn't match - go back to finding another 'trade' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Extracting info from 'trade' MESSAGE transaction */ + labelTradeTxnExtract = codeByteBuffer.position(); + + // Extract message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); + + // Extract trade partner's Dogecoin public key hash (PKH) from message into B + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerDogecoinPKHOffset)); + // Store partner's Dogecoin PKH (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerDogecoinPKHPointer)); + // Extract AT trade timeout (minutes) (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrRefundTimeout)); + + // Grab next 32 bytes + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); + + // Extract hash-of-secret-A (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); + // Extract lockTime-A (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); + + // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); + + /* We are in 'trade mode' */ + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ + + // Fetch current block 'timestamp' + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); + // If we're not past refund 'timestamp' then look for next transaction + codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + // We're past refund 'timestamp' so go refund everything back to AT creator + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); + + /* Transaction processing loop */ + labelRedeemTxnLoop = codeByteBuffer.position(); + + /* Sleep until message arrives */ + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.SLEEP_UNTIL_MESSAGE.value, addrLastTxnTimestamp)); + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckRedeemTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check message payload length */ + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to sender checking code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); + // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Check transaction's sender */ + labelCheckRedeemTxnSender = codeByteBuffer.position(); + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check 'secret-A' in transaction's message */ + + // Extract secret-A from first 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); + // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); + // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). + // Save the equality result (1 if they match, 0 otherwise) into addrResult. + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); + // If hashes don't match, addrResult will be zero so go find another transaction + codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Success! Pay arranged amount to receiving address */ + labelPayout = codeByteBuffer.position(); + + // Extract Qortal receiving address from next 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); + // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); + // Pay AT's balance to receiving address + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); + // Set redeemed mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + // Fall-through to refunding any remaining balance back to AT creator + + /* Refund balance back to AT creator */ + labelRefund = codeByteBuffer.position(); + + // Set refunded mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile DOGE-QORT ACCT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + assert Arrays.equals(Crypto.digest(codeBytes), DogecoinACCTv2.CODE_BYTES_HASH) + : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { + ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + byte[] addressBytes = new byte[25]; // for general use + String atAddress = atStateData.getATAddress(); + + CrossChainTradeData tradeData = new CrossChainTradeData(); + + tradeData.foreignBlockchain = SupportedBlockchain.DOGECOIN.name(); + tradeData.acctName = NAME; + + tradeData.qortalAtAddress = atAddress; + tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); + tradeData.creationTimestamp = creationTimestamp; + + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + + byte[] stateData = atStateData.getStateData(); + ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); + dataByteBuffer.position(MachineState.HEADER_LENGTH); + + /* Constants */ + + // Skip creator's trade address + dataByteBuffer.get(addressBytes); + tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Creator's Dogecoin/foreign public key hash + tradeData.creatorForeignPKH = new byte[20]; + dataByteBuffer.get(tradeData.creatorForeignPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes + + // We don't use secret-B + tradeData.hashOfSecretB = null; + + // Redeem payout + tradeData.qortAmount = dataByteBuffer.getLong(); + + // Expected DOGE amount + tradeData.expectedForeignAmount = dataByteBuffer.getLong(); + + // Trade timeout + tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); + + // Skip MESSAGE transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'trade' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'redeem' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Qortal trade address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message sender + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for partner's Dogecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Dogecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'redeem' message data offset for partner's Qortal receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message data + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip message data length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + /* End of constants / begin variables */ + + // Skip AT creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Partner's trade address (if present) + dataByteBuffer.get(addressBytes); + String qortalRecipient = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Potential lockTimeA (if in trade mode) + int lockTimeA = (int) dataByteBuffer.getLong(); + + // AT refund timeout (probably only useful for debugging) + int refundTimeout = (int) dataByteBuffer.getLong(); + + // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) + long tradeRefundTimestamp = dataByteBuffer.getLong(); + + // Skip last transaction timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip block timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary result + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message sender + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Skip message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message data + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Potential hash160 of secret A + byte[] hashOfSecretA = new byte[20]; + dataByteBuffer.get(hashOfSecretA); + dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes + + // Potential partner's Dogecoin PKH + byte[] partnerDogecoinPKH = new byte[20]; + dataByteBuffer.get(partnerDogecoinPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerDogecoinPKH.length); // skip to 32 bytes + + // Partner's receiving address (if present) + byte[] partnerReceivingAddress = new byte[25]; + dataByteBuffer.get(partnerReceivingAddress); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes + + // Trade AT's 'mode' + long modeValue = dataByteBuffer.getLong(); + AcctMode mode = AcctMode.valueOf((int) (modeValue & 0xffL)); + + /* End of variables */ + + if (mode != null && mode != AcctMode.OFFERING) { + tradeData.mode = mode; + tradeData.refundTimeout = refundTimeout; + tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; + tradeData.qortalPartnerAddress = qortalRecipient; + tradeData.hashOfSecretA = hashOfSecretA; + tradeData.partnerForeignPKH = partnerDogecoinPKH; + tradeData.lockTimeA = lockTimeA; + + if (mode == AcctMode.REDEEMED) + tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); + } else { + tradeData.mode = AcctMode.OFFERING; + } + + tradeData.duplicateDeprecated(); + + return tradeData; + } + + /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ + public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); + } + + /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ + public static OfferMessageData extractOfferMessageData(byte[] messageData) { + if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) + return null; + + OfferMessageData offerMessageData = new OfferMessageData(); + offerMessageData.partnerDogecoinPKH = Arrays.copyOfRange(messageData, 0, 20); + offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); + offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); + + return offerMessageData; + } + + /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ + public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int refundTimeout) { + byte[] data = new byte[TRADE_MESSAGE_LENGTH]; + byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + byte[] refundTimeoutBytes = BitTwiddling.toBEByteArray((long) refundTimeout); + + System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); + System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); + System.arraycopy(refundTimeoutBytes, 0, data, 56, refundTimeoutBytes.length); + System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); + System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); + + return data; + } + + /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ + @Override + public byte[] buildCancelMessage(String creatorQortalAddress) { + byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; + byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); + + System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); + + return data; + } + + /** Returns 'redeem' MESSAGE payload for trade partner to send to AT. */ + public static byte[] buildRedeemMessage(byte[] secretA, String qortalReceivingAddress) { + byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; + byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); + + System.arraycopy(secretA, 0, data, 0, secretA.length); + System.arraycopy(qortalReceivingAddressBytes, 0, data, 32, qortalReceivingAddressBytes.length); + + return data; + } + + /** Returns refund timeout (minutes) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ + public static int calcRefundTimeout(long offerMessageTimestamp, int lockTimeA) { + // refund should be triggered halfway between offerMessageTimestamp and lockTimeA + return (int) ((lockTimeA - (offerMessageTimestamp / 1000L)) / 2L / 60L); + } + + @Override + public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { + String atAddress = crossChainTradeData.qortalAtAddress; + String redeemerAddress = crossChainTradeData.qortalPartnerAddress; + + // We don't have partner's public key so we check every message to AT + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); + if (messageTransactionsData == null) + return null; + + // Find 'redeem' message + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + // Check message payload type/encryption + if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) + continue; + + // Check message payload size + byte[] messageData = messageTransactionData.getData(); + if (messageData.length != REDEEM_MESSAGE_LENGTH) + // Wrong payload length + continue; + + // Check sender + if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) + // Wrong sender; + continue; + + // Extract secretA + byte[] secretA = new byte[32]; + System.arraycopy(messageData, 0, secretA, 0, secretA.length); + + byte[] hashOfSecretA = Crypto.hash160(secretA); + if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) + continue; + + return secretA; + } + + return null; + } + +} diff --git a/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java b/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java new file mode 100644 index 000000000..002a4448f --- /dev/null +++ b/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java @@ -0,0 +1,858 @@ +package org.qortal.crosschain; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ciyam.at.*; +import org.qortal.account.Account; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.utils.Base58; +import org.qortal.utils.BitTwiddling; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import static org.ciyam.at.OpCode.calcOffset; + +/** + * Cross-chain trade AT + * + *

+ *

    + *
  • Bob generates Dogecoin & Qortal 'trade' keys + *
      + *
    • private key required to sign P2SH redeem tx
      • + *
    • private key could be used to create 'secret' (e.g. double-SHA256)
      • + *
    • encrypted private key could be stored in Qortal AT for access by Bob from any node
      • + *
    + *
    • + *
  • Bob deploys Qortal AT + *
      + *
    + *
    • + *
  • Alice finds Qortal AT and wants to trade + *
      + *
    • Alice generates Dogecoin & Qortal 'trade' keys
      • + *
    • Alice funds Dogecoin P2SH-A
      • + *
    • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: + *
        + *
      • hash-of-secret-A
        • + *
      • her 'trade' Dogecoin PKH
        • + *
      + *
      • + *
    + *
    • + *
  • Bob receives "offer" MESSAGE + *
      + *
    • Checks Alice's P2SH-A
      • + *
    • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: + *
        + *
      • Alice's trade Qortal address
        • + *
      • Alice's trade Dogecoin PKH
        • + *
      • hash-of-secret-A
        • + *
      + *
      • + *
    + *
    • + *
  • Alice checks Qortal AT to confirm it's locked to her + *
      + *
    • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: + *
        + *
      • secret-A
        • + *
      • Qortal receiving address of her chosing
        • + *
      + *
      • + *
    • AT's QORT funds are sent to Qortal receiving address
      • + *
    + *
    • + *
  • Bob checks AT, extracts secret-A + *
      + *
    • Bob redeems P2SH-A using his Dogecoin trade key and secret-A
      • + *
    • P2SH-A DOGE funds end up at Dogecoin address determined by redeem transaction output(s)
      • + *
    + *
    • + *
+ */ +public class DogecoinACCTv3 implements ACCT { + + private static final Logger LOGGER = LogManager.getLogger(DogecoinACCTv3.class); + + public static final String NAME = DogecoinACCTv3.class.getSimpleName(); + public static final byte[] CODE_BYTES_HASH = HashCode.fromString("03b44087f6325463eb745aa32d9a782add03148bcfbe73ffd8854ce55ff863d4").asBytes(); // SHA256 of AT code bytes + + public static final int SECRET_LENGTH = 32; + + /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ + private static final int MODE_VALUE_OFFSET = 61; + /** Byte offset into AT state data where 'mode' variable (long) is stored. */ + public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); + + public static class OfferMessageData { + public byte[] partnerDogecoinPKH; + public byte[] hashOfSecretA; + public long lockTimeA; + } + public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerDogecoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; + public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ + + 24 /*partner's Dogecoin PKH (padded from 20 to 24)*/ + + 8 /*AT trade timeout (minutes)*/ + + 24 /*hash of secret-A (padded from 20 to 24)*/ + + 8 /*lockTimeA*/; + public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret-A*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; + public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; + + private static DogecoinACCTv3 instance; + + private DogecoinACCTv3() { + } + + public static synchronized DogecoinACCTv3 getInstance() { + if (instance == null) + instance = new DogecoinACCTv3(); + + return instance; + } + + @Override + public byte[] getCodeBytesHash() { + return CODE_BYTES_HASH; + } + + @Override + public int getModeByteOffset() { + return MODE_BYTE_OFFSET; + } + + @Override + public ForeignBlockchain getBlockchain() { + return Dogecoin.getInstance(); + } + + /** + * Returns Qortal AT creation bytes for cross-chain trading AT. + *

+ * tradeTimeout (minutes) is the time window for the trade partner to send the + * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. + * + * @param creatorTradeAddress AT creator's trade Qortal address + * @param dogecoinPublicKeyHash 20-byte HASH160 of creator's trade Dogecoin public key + * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT + * @param dogecoinAmount how much DOGE the AT creator is expecting to trade + * @param tradeTimeout suggested timeout for entire trade + */ + public static byte[] buildQortalAT(String creatorTradeAddress, byte[] dogecoinPublicKeyHash, long qortAmount, long dogecoinAmount, int tradeTimeout) { + if (dogecoinPublicKeyHash.length != 20) + throw new IllegalArgumentException("Dogecoin public key hash should be 20 bytes"); + + // Labels for data segment addresses + int addrCounter = 0; + + // Constants (with corresponding dataByteBuffer.put*() calls below) + + final int addrCreatorTradeAddress1 = addrCounter++; + final int addrCreatorTradeAddress2 = addrCounter++; + final int addrCreatorTradeAddress3 = addrCounter++; + final int addrCreatorTradeAddress4 = addrCounter++; + + final int addrDogecoinPublicKeyHash = addrCounter; + addrCounter += 4; + + final int addrQortAmount = addrCounter++; + final int addrDogecoinAmount = addrCounter++; + final int addrTradeTimeout = addrCounter++; + + final int addrMessageTxnType = addrCounter++; + final int addrExpectedTradeMessageLength = addrCounter++; + final int addrExpectedRedeemMessageLength = addrCounter++; + + final int addrCreatorAddressPointer = addrCounter++; + final int addrQortalPartnerAddressPointer = addrCounter++; + final int addrMessageSenderPointer = addrCounter++; + + final int addrTradeMessagePartnerDogecoinPKHOffset = addrCounter++; + final int addrPartnerDogecoinPKHPointer = addrCounter++; + final int addrTradeMessageHashOfSecretAOffset = addrCounter++; + final int addrHashOfSecretAPointer = addrCounter++; + + final int addrRedeemMessageReceivingAddressOffset = addrCounter++; + + final int addrMessageDataPointer = addrCounter++; + final int addrMessageDataLength = addrCounter++; + + final int addrPartnerReceivingAddressPointer = addrCounter++; + + final int addrEndOfConstants = addrCounter; + + // Variables + + final int addrCreatorAddress1 = addrCounter++; + final int addrCreatorAddress2 = addrCounter++; + final int addrCreatorAddress3 = addrCounter++; + final int addrCreatorAddress4 = addrCounter++; + + final int addrQortalPartnerAddress1 = addrCounter++; + final int addrQortalPartnerAddress2 = addrCounter++; + final int addrQortalPartnerAddress3 = addrCounter++; + final int addrQortalPartnerAddress4 = addrCounter++; + + final int addrLockTimeA = addrCounter++; + final int addrRefundTimeout = addrCounter++; + final int addrRefundTimestamp = addrCounter++; + final int addrLastTxnTimestamp = addrCounter++; + final int addrBlockTimestamp = addrCounter++; + final int addrTxnType = addrCounter++; + final int addrResult = addrCounter++; + + final int addrMessageSender1 = addrCounter++; + final int addrMessageSender2 = addrCounter++; + final int addrMessageSender3 = addrCounter++; + final int addrMessageSender4 = addrCounter++; + + final int addrMessageLength = addrCounter++; + + final int addrMessageData = addrCounter; + addrCounter += 4; + + final int addrHashOfSecretA = addrCounter; + addrCounter += 4; + + final int addrPartnerDogecoinPKH = addrCounter; + addrCounter += 4; + + final int addrPartnerReceivingAddress = addrCounter; + addrCounter += 4; + + final int addrMode = addrCounter++; + assert addrMode == MODE_VALUE_OFFSET : String.format("addrMode %d does not match MODE_VALUE_OFFSET %d", addrMode, MODE_VALUE_OFFSET); + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // AT creator's trade Qortal address, decoded from Base58 + assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; + byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); + dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); + + // Dogecoin public key hash + assert dataByteBuffer.position() == addrDogecoinPublicKeyHash * MachineState.VALUE_SIZE : "addrDogecoinPublicKeyHash incorrect"; + dataByteBuffer.put(Bytes.ensureCapacity(dogecoinPublicKeyHash, 32, 0)); + + // Redeem Qort amount + assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; + dataByteBuffer.putLong(qortAmount); + + // Expected Dogecoin amount + assert dataByteBuffer.position() == addrDogecoinAmount * MachineState.VALUE_SIZE : "addrDogecoinAmount incorrect"; + dataByteBuffer.putLong(dogecoinAmount); + + // Suggested trade timeout (minutes) + assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; + dataByteBuffer.putLong(tradeTimeout); + + // We're only interested in MESSAGE transactions + assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; + dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); + + // Expected length of 'trade' MESSAGE data from AT creator + assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; + dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); + + // Expected length of 'redeem' MESSAGE data from trade partner + assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; + dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); + + // Index into data segment of AT creator's address, used by GET_B_IND + assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; + dataByteBuffer.putLong(addrCreatorAddress1); + + // Index into data segment of partner's Qortal address, used by SET_B_IND + assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; + dataByteBuffer.putLong(addrQortalPartnerAddress1); + + // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND + assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; + dataByteBuffer.putLong(addrMessageSender1); + + // Offset into 'trade' MESSAGE data payload for extracting partner's Dogecoin PKH + assert dataByteBuffer.position() == addrTradeMessagePartnerDogecoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerDogecoinPKHOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Index into data segment of partner's Dogecoin PKH, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerDogecoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerDogecoinPKHPointer incorrect"; + dataByteBuffer.putLong(addrPartnerDogecoinPKH); + + // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A + assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; + dataByteBuffer.putLong(64L); + + // Index into data segment to hash of secret A, used by GET_B_IND + assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; + dataByteBuffer.putLong(addrHashOfSecretA); + + // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address + assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Source location and length for hashing any passed secret + assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; + dataByteBuffer.putLong(addrMessageData); + assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; + dataByteBuffer.putLong(32L); + + // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; + dataByteBuffer.putLong(addrPartnerReceivingAddress); + + assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; + + // Code labels + Integer labelRefund = null; + + Integer labelTradeTxnLoop = null; + Integer labelCheckTradeTxn = null; + Integer labelCheckCancelTxn = null; + Integer labelNotTradeNorCancelTxn = null; + Integer labelCheckNonRefundTradeTxn = null; + Integer labelTradeTxnExtract = null; + Integer labelRedeemTxnLoop = null; + Integer labelCheckRedeemTxn = null; + Integer labelCheckRedeemTxnSender = null; + Integer labelPayout = null; + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + /* NOP - to ensure DOGECOIN ACCT is unique */ + codeByteBuffer.put(OpCode.NOP.compile()); + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); + + // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ + + /* Transaction processing loop */ + labelTradeTxnLoop = codeByteBuffer.position(); + + /* Sleep until message arrives */ + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.SLEEP_UNTIL_MESSAGE.value, addrLastTxnTimestamp)); + + // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckTradeTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); + + /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + // Message sender's address matches AT creator's trade address so go process 'trade' message + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); + + /* Checking message sender for possible cancel message */ + labelCheckCancelTxn = codeByteBuffer.position(); + + // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + // Partner address is AT creator's address, so cancel offer and finish. + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + /* Not trade nor cancel message */ + labelNotTradeNorCancelTxn = codeByteBuffer.position(); + + // Loop to find another transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Possible switch-to-trade-mode message */ + labelCheckNonRefundTradeTxn = codeByteBuffer.position(); + + // Check 'trade' message we received has expected number of message bytes + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to info extraction code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); + // Message length didn't match - go back to finding another 'trade' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Extracting info from 'trade' MESSAGE transaction */ + labelTradeTxnExtract = codeByteBuffer.position(); + + // Extract message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); + + // Extract trade partner's Dogecoin public key hash (PKH) from message into B + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerDogecoinPKHOffset)); + // Store partner's Dogecoin PKH (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerDogecoinPKHPointer)); + // Extract AT trade timeout (minutes) (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrRefundTimeout)); + + // Grab next 32 bytes + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); + + // Extract hash-of-secret-A (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); + // Extract lockTime-A (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); + + // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); + + /* We are in 'trade mode' */ + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ + + // Fetch current block 'timestamp' + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); + // If we're not past refund 'timestamp' then look for next transaction + codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + // We're past refund 'timestamp' so go refund everything back to AT creator + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); + + /* Transaction processing loop */ + labelRedeemTxnLoop = codeByteBuffer.position(); + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckRedeemTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check message payload length */ + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to sender checking code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); + // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Check transaction's sender */ + labelCheckRedeemTxnSender = codeByteBuffer.position(); + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check 'secret-A' in transaction's message */ + + // Extract secret-A from first 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); + // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); + // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). + // Save the equality result (1 if they match, 0 otherwise) into addrResult. + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); + // If hashes don't match, addrResult will be zero so go find another transaction + codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Success! Pay arranged amount to receiving address */ + labelPayout = codeByteBuffer.position(); + + // Extract Qortal receiving address from next 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); + // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); + // Pay AT's balance to receiving address + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); + // Set redeemed mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + // Fall-through to refunding any remaining balance back to AT creator + + /* Refund balance back to AT creator */ + labelRefund = codeByteBuffer.position(); + + // Set refunded mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile DOGE-QORT ACCT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + assert Arrays.equals(Crypto.digest(codeBytes), DogecoinACCTv3.CODE_BYTES_HASH) + : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { + ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + byte[] addressBytes = new byte[25]; // for general use + String atAddress = atStateData.getATAddress(); + + CrossChainTradeData tradeData = new CrossChainTradeData(); + + tradeData.foreignBlockchain = SupportedBlockchain.DOGECOIN.name(); + tradeData.acctName = NAME; + + tradeData.qortalAtAddress = atAddress; + tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); + tradeData.creationTimestamp = creationTimestamp; + + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + + byte[] stateData = atStateData.getStateData(); + ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); + dataByteBuffer.position(MachineState.HEADER_LENGTH); + + /* Constants */ + + // Skip creator's trade address + dataByteBuffer.get(addressBytes); + tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Creator's Dogecoin/foreign public key hash + tradeData.creatorForeignPKH = new byte[20]; + dataByteBuffer.get(tradeData.creatorForeignPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes + + // We don't use secret-B + tradeData.hashOfSecretB = null; + + // Redeem payout + tradeData.qortAmount = dataByteBuffer.getLong(); + + // Expected DOGE amount + tradeData.expectedForeignAmount = dataByteBuffer.getLong(); + + // Trade timeout + tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); + + // Skip MESSAGE transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'trade' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'redeem' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Qortal trade address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message sender + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for partner's Dogecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Dogecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'redeem' message data offset for partner's Qortal receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message data + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip message data length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + /* End of constants / begin variables */ + + // Skip AT creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Partner's trade address (if present) + dataByteBuffer.get(addressBytes); + String qortalRecipient = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Potential lockTimeA (if in trade mode) + int lockTimeA = (int) dataByteBuffer.getLong(); + + // AT refund timeout (probably only useful for debugging) + int refundTimeout = (int) dataByteBuffer.getLong(); + + // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) + long tradeRefundTimestamp = dataByteBuffer.getLong(); + + // Skip last transaction timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip block timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary result + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message sender + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Skip message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message data + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Potential hash160 of secret A + byte[] hashOfSecretA = new byte[20]; + dataByteBuffer.get(hashOfSecretA); + dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes + + // Potential partner's Dogecoin PKH + byte[] partnerDogecoinPKH = new byte[20]; + dataByteBuffer.get(partnerDogecoinPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerDogecoinPKH.length); // skip to 32 bytes + + // Partner's receiving address (if present) + byte[] partnerReceivingAddress = new byte[25]; + dataByteBuffer.get(partnerReceivingAddress); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes + + // Trade AT's 'mode' + long modeValue = dataByteBuffer.getLong(); + AcctMode mode = AcctMode.valueOf((int) (modeValue & 0xffL)); + + /* End of variables */ + + if (mode != null && mode != AcctMode.OFFERING) { + tradeData.mode = mode; + tradeData.refundTimeout = refundTimeout; + tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; + tradeData.qortalPartnerAddress = qortalRecipient; + tradeData.hashOfSecretA = hashOfSecretA; + tradeData.partnerForeignPKH = partnerDogecoinPKH; + tradeData.lockTimeA = lockTimeA; + + if (mode == AcctMode.REDEEMED) + tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); + } else { + tradeData.mode = AcctMode.OFFERING; + } + + tradeData.duplicateDeprecated(); + + return tradeData; + } + + /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ + public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); + } + + /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ + public static OfferMessageData extractOfferMessageData(byte[] messageData) { + if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) + return null; + + OfferMessageData offerMessageData = new OfferMessageData(); + offerMessageData.partnerDogecoinPKH = Arrays.copyOfRange(messageData, 0, 20); + offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); + offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); + + return offerMessageData; + } + + /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ + public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int refundTimeout) { + byte[] data = new byte[TRADE_MESSAGE_LENGTH]; + byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + byte[] refundTimeoutBytes = BitTwiddling.toBEByteArray((long) refundTimeout); + + System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); + System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); + System.arraycopy(refundTimeoutBytes, 0, data, 56, refundTimeoutBytes.length); + System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); + System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); + + return data; + } + + /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ + @Override + public byte[] buildCancelMessage(String creatorQortalAddress) { + byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; + byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); + + System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); + + return data; + } + + /** Returns 'redeem' MESSAGE payload for trade partner to send to AT. */ + public static byte[] buildRedeemMessage(byte[] secretA, String qortalReceivingAddress) { + byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; + byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); + + System.arraycopy(secretA, 0, data, 0, secretA.length); + System.arraycopy(qortalReceivingAddressBytes, 0, data, 32, qortalReceivingAddressBytes.length); + + return data; + } + + /** Returns refund timeout (minutes) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ + public static int calcRefundTimeout(long offerMessageTimestamp, int lockTimeA) { + // refund should be triggered halfway between offerMessageTimestamp and lockTimeA + return (int) ((lockTimeA - (offerMessageTimestamp / 1000L)) / 2L / 60L); + } + + @Override + public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { + String atAddress = crossChainTradeData.qortalAtAddress; + String redeemerAddress = crossChainTradeData.qortalPartnerAddress; + + // We don't have partner's public key so we check every message to AT + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); + if (messageTransactionsData == null) + return null; + + // Find 'redeem' message + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + // Check message payload type/encryption + if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) + continue; + + // Check message payload size + byte[] messageData = messageTransactionData.getData(); + if (messageData.length != REDEEM_MESSAGE_LENGTH) + // Wrong payload length + continue; + + // Check sender + if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) + // Wrong sender; + continue; + + // Extract secretA + byte[] secretA = new byte[32]; + System.arraycopy(messageData, 0, secretA, 0, secretA.length); + + byte[] hashOfSecretA = Crypto.hash160(secretA); + if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) + continue; + + return secretA; + } + + return null; + } + +} diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index 41c3d99d7..7f1eb4c49 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -1,21 +1,14 @@ package org.qortal.crosschain; import java.io.IOException; +import java.math.BigDecimal; import java.net.InetSocketAddress; import java.net.Socket; import java.net.SocketAddress; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Random; -import java.util.Scanner; -import java.util.Set; - -import javax.net.ssl.SSLSocket; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import javax.net.ssl.SSLSocketFactory; import org.apache.logging.log4j.LogManager; @@ -25,30 +18,37 @@ import org.json.simple.JSONValue; import org.qortal.crypto.Crypto; import org.qortal.crypto.TrustlessSSLSocketFactory; -import org.qortal.utils.Pair; import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; +import org.qortal.utils.BitTwiddling; -public class ElectrumX { +/** ElectrumX network support for querying Bitcoiny-related info like block headers, transaction outputs, etc. */ +public class ElectrumX extends BitcoinyBlockchainProvider { private static final Logger LOGGER = LogManager.getLogger(ElectrumX.class); private static final Random RANDOM = new Random(); - private static final int DEFAULT_TCP_PORT = 50001; - private static final int DEFAULT_SSL_PORT = 50002; - + private static final double MIN_PROTOCOL_VERSION = 1.2; private static final int BLOCK_HEADER_LENGTH = 80; - private static final Map instances = new HashMap<>(); + // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'})" + private static final Pattern DAEMON_ERROR_REGEX = Pattern.compile("DaemonError\\(\\{.*'code': ?(-?[0-9]+).*\\}\\)\\z"); // Capture 'code' inside curly-brace content + + /** Error message sent by some ElectrumX servers when they don't support returning verbose transactions. */ + private static final String VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE = "verbose transactions are currently unsupported"; + + private static final int RESPONSE_TIME_READINGS = 5; + private static final long MAX_AVG_RESPONSE_TIME = 500L; // ms - static class Server { + public static class Server { String hostname; - enum ConnectionType { TCP, SSL }; + public enum ConnectionType { TCP, SSL } ConnectionType connectionType; int port; + private List responseTimes = new ArrayList<>(); public Server(String hostname, ConnectionType connectionType, int port) { this.hostname = hostname; @@ -56,6 +56,25 @@ public Server(String hostname, ConnectionType connectionType, int port) { this.port = port; } + public void addResponseTime(long responseTime) { + while (this.responseTimes.size() > RESPONSE_TIME_READINGS) { + this.responseTimes.remove(0); + } + this.responseTimes.add(responseTime); + } + + public long averageResponseTime() { + if (this.responseTimes.size() < RESPONSE_TIME_READINGS) { + // Not enough readings yet + return 0L; + } + OptionalDouble average = this.responseTimes.stream().mapToDouble(a -> a).average(); + if (average.isPresent()) { + return Double.valueOf(average.getAsDouble()).longValue(); + } + return 0L; + } + @Override public boolean equals(Object other) { if (other == this) @@ -82,169 +101,395 @@ public String toString() { } } private Set servers = new HashSet<>(); + private List remainingServers = new ArrayList<>(); + private Set uselessServers = Collections.synchronizedSet(new HashSet<>()); + + private final String netId; + private final String expectedGenesisHash; + private final Map defaultPorts = new EnumMap<>(Server.ConnectionType.class); + private final Object serverLock = new Object(); private Server currentServer; private Socket socket; private Scanner scanner; private int nextId = 1; - // Constructors - - private ElectrumX(String bitcoinNetwork) { - switch (bitcoinNetwork) { - case "MAIN": - servers.addAll(Arrays.asList()); - break; - - case "TEST3": - servers.addAll(Arrays.asList( - new Server("tn.not.fyi", Server.ConnectionType.TCP, 55001), - new Server("tn.not.fyi", Server.ConnectionType.SSL, 55002), - new Server("testnet.aranguren.org", Server.ConnectionType.TCP, 51001), - new Server("testnet.aranguren.org", Server.ConnectionType.SSL, 51002), - new Server("testnet.hsmiths.com", Server.ConnectionType.SSL, 53012))); - break; - - case "REGTEST": - servers.addAll(Arrays.asList( - new Server("localhost", Server.ConnectionType.TCP, DEFAULT_TCP_PORT), - new Server("localhost", Server.ConnectionType.SSL, DEFAULT_SSL_PORT))); - break; - - default: - throw new IllegalArgumentException(String.format("Bitcoin network '%s' unknown", bitcoinNetwork)); + private static final int TX_CACHE_SIZE = 1000; + @SuppressWarnings("serial") + private final Map transactionCache = Collections.synchronizedMap(new LinkedHashMap<>(TX_CACHE_SIZE + 1, 0.75F, true) { + // This method is called just after a new entry has been added + @Override + public boolean removeEldestEntry(Map.Entry eldest) { + return size() > TX_CACHE_SIZE; } + }); + + // Constructors - LOGGER.debug(() -> String.format("Starting ElectrumX support for %s Bitcoin network", bitcoinNetwork)); - rpc("server.banner"); + public ElectrumX(String netId, String genesisHash, Collection initialServerList, Map defaultPorts) { + this.netId = netId; + this.expectedGenesisHash = genesisHash; + this.servers.addAll(initialServerList); + this.defaultPorts.putAll(defaultPorts); } - public static synchronized ElectrumX getInstance(String bitcoinNetwork) { - if (!instances.containsKey(bitcoinNetwork)) - instances.put(bitcoinNetwork, new ElectrumX(bitcoinNetwork)); + // Methods for use by other classes - return instances.get(bitcoinNetwork); + @Override + public String getNetId() { + return this.netId; } - // Methods for use by other classes + /** + * Returns current blockchain height. + *

+ * @throws ForeignBlockchainException if error occurs + */ + @Override + public int getCurrentHeight() throws ForeignBlockchainException { + Object blockObj = this.rpc("blockchain.headers.subscribe"); + if (!(blockObj instanceof JSONObject)) + throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.headers.subscribe RPC"); - public Integer getCurrentHeight() { - JSONObject blockJson = (JSONObject) this.rpc("blockchain.headers.subscribe"); - if (blockJson == null || !blockJson.containsKey("height")) - return null; + JSONObject blockJson = (JSONObject) blockObj; - return ((Long) blockJson.get("height")).intValue(); + Object heightObj = blockJson.get("height"); + + if (!(heightObj instanceof Long)) + throw new ForeignBlockchainException.NetworkException("Missing/invalid 'height' in JSON from ElectrumX blockchain.headers.subscribe RPC"); + + return ((Long) heightObj).intValue(); } - public List getBlockHeaders(int startHeight, long count) { - JSONObject blockJson = (JSONObject) this.rpc("blockchain.block.headers", startHeight, count); - if (blockJson == null || !blockJson.containsKey("count") || !blockJson.containsKey("hex")) - return null; + /** + * Returns list of raw block headers, starting from startHeight inclusive. + *

+ * @throws ForeignBlockchainException if error occurs + */ + @Override + public List getRawBlockHeaders(int startHeight, int count) throws ForeignBlockchainException { + Object blockObj = this.rpc("blockchain.block.headers", startHeight, count); + if (!(blockObj instanceof JSONObject)) + throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.block.headers RPC"); - Long returnedCount = (Long) blockJson.get("count"); - String hex = (String) blockJson.get("hex"); + JSONObject blockJson = (JSONObject) blockObj; - byte[] raw = HashCode.fromString(hex).asBytes(); - if (raw.length != returnedCount * BLOCK_HEADER_LENGTH) - return null; + Object countObj = blockJson.get("count"); + Object hexObj = blockJson.get("hex"); + + if (!(countObj instanceof Long) || !(hexObj instanceof String)) + throw new ForeignBlockchainException.NetworkException("Missing/invalid 'count' or 'hex' entries in JSON from ElectrumX blockchain.block.headers RPC"); + + Long returnedCount = (Long) countObj; + String hex = (String) hexObj; List rawBlockHeaders = new ArrayList<>(returnedCount.intValue()); - for (int i = 0; i < returnedCount; ++i) - rawBlockHeaders.add(Arrays.copyOfRange(raw, i * BLOCK_HEADER_LENGTH, (i + 1) * BLOCK_HEADER_LENGTH)); + + byte[] raw = HashCode.fromString(hex).asBytes(); + + // Most chains use a fixed length 80 byte header, so block headers can be split up by dividing the hex into + // 80-byte segments. However, some chains such as DOGE use variable length headers due to AuxPoW or other + // reasons. In these cases we can identify the start of each block header by the location of the block version + // numbers. Each block starts with a version number, and for DOGE this is easily identifiable (6422788) at the + // time of writing (Jul 2021). If we encounter a chain that is using more generic version numbers (e.g. 1) + // and can't be used to accurately identify block indexes, then there are sufficient checks to ensure an + // exception is thrown. + + if (raw.length == returnedCount * BLOCK_HEADER_LENGTH) { + // Fixed-length header (BTC, LTC, etc) + for (int i = 0; i < returnedCount; ++i) { + rawBlockHeaders.add(Arrays.copyOfRange(raw, i * BLOCK_HEADER_LENGTH, (i + 1) * BLOCK_HEADER_LENGTH)); + } + } + else if (raw.length > returnedCount * BLOCK_HEADER_LENGTH) { + // Assume AuxPoW variable length header (DOGE) + int referenceVersion = BitTwiddling.intFromLEBytes(raw, 0); // DOGE uses 6422788 at time of commit (Jul 2021) + for (int i = 0; i < raw.length - 4; ++i) { + // Locate the start of each block by its version number + if (BitTwiddling.intFromLEBytes(raw, i) == referenceVersion) { + rawBlockHeaders.add(Arrays.copyOfRange(raw, i, i + BLOCK_HEADER_LENGTH)); + } + } + // Ensure that we found the correct number of block headers + if (rawBlockHeaders.size() != count) { + throw new ForeignBlockchainException.NetworkException("Unexpected raw header contents in JSON from ElectrumX blockchain.block.headers RPC."); + } + } + else if (raw.length != returnedCount * BLOCK_HEADER_LENGTH) { + throw new ForeignBlockchainException.NetworkException("Unexpected raw header length in JSON from ElectrumX blockchain.block.headers RPC"); + } return rawBlockHeaders; } - public Long getBalance(byte[] script) { + /** + * Returns confirmed balance, based on passed payment script. + *

+ * @return confirmed balance, or zero if script unknown + * @throws ForeignBlockchainException if there was an error + */ + @Override + public long getConfirmedBalance(byte[] script) throws ForeignBlockchainException { byte[] scriptHash = Crypto.digest(script); Bytes.reverse(scriptHash); - JSONObject balanceJson = (JSONObject) this.rpc("blockchain.scripthash.get_balance", HashCode.fromBytes(scriptHash).toString()); - if (balanceJson == null || !balanceJson.containsKey("confirmed")) - return null; + Object balanceObj = this.rpc("blockchain.scripthash.get_balance", HashCode.fromBytes(scriptHash).toString()); + if (!(balanceObj instanceof JSONObject)) + throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.scripthash.get_balance RPC"); + + JSONObject balanceJson = (JSONObject) balanceObj; + + Object confirmedBalanceObj = balanceJson.get("confirmed"); + + if (!(confirmedBalanceObj instanceof Long)) + throw new ForeignBlockchainException.NetworkException("Missing confirmed balance from ElectrumX blockchain.scripthash.get_balance RPC"); return (Long) balanceJson.get("confirmed"); } - public List> getUnspentOutputs(byte[] script) { + /** + * Returns list of unspent outputs pertaining to passed payment script. + *

+ * @return list of unspent outputs, or empty list if script unknown + * @throws ForeignBlockchainException if there was an error. + */ + @Override + public List getUnspentOutputs(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException { byte[] scriptHash = Crypto.digest(script); Bytes.reverse(scriptHash); - JSONArray unspentJson = (JSONArray) this.rpc("blockchain.scripthash.listunspent", HashCode.fromBytes(scriptHash).toString()); - if (unspentJson == null) - return null; + Object unspentJson = this.rpc("blockchain.scripthash.listunspent", HashCode.fromBytes(scriptHash).toString()); + if (!(unspentJson instanceof JSONArray)) + throw new ForeignBlockchainException("Expected array output from ElectrumX blockchain.scripthash.listunspent RPC"); - List> unspentOutputs = new ArrayList<>(); - for (Object rawUnspent : unspentJson) { + List unspentOutputs = new ArrayList<>(); + for (Object rawUnspent : (JSONArray) unspentJson) { JSONObject unspent = (JSONObject) rawUnspent; + int height = ((Long) unspent.get("height")).intValue(); + // We only want unspent outputs from confirmed transactions (and definitely not mempool duplicates with height 0) + if (!includeUnconfirmed && height <= 0) + continue; + byte[] txHash = HashCode.fromString((String) unspent.get("tx_hash")).asBytes(); int outputIndex = ((Long) unspent.get("tx_pos")).intValue(); + long value = (Long) unspent.get("value"); - unspentOutputs.add(new Pair<>(txHash, outputIndex)); + unspentOutputs.add(new UnspentOutput(txHash, outputIndex, height, value)); } return unspentOutputs; } - public byte[] getRawTransaction(byte[] txHash) { - String rawTransactionHex = (String) this.rpc("blockchain.transaction.get", HashCode.fromBytes(txHash).toString()); - if (rawTransactionHex == null) - return null; + /** + * Returns raw transaction for passed transaction hash. + *

+ * NOTE: Do not mutate returned byte[]! + * + * @throws ForeignBlockchainException.NotFoundException if transaction not found + * @throws ForeignBlockchainException if error occurs + */ + @Override + public byte[] getRawTransaction(String txHash) throws ForeignBlockchainException { + Object rawTransactionHex; + try { + rawTransactionHex = this.rpc("blockchain.transaction.get", txHash, false); + } catch (ForeignBlockchainException.NetworkException e) { + // DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'}) + if (Integer.valueOf(-5).equals(e.getDaemonErrorCode())) + throw new ForeignBlockchainException.NotFoundException(e.getMessage()); + + throw e; + } + + if (!(rawTransactionHex instanceof String)) + throw new ForeignBlockchainException.NetworkException("Expected hex string as raw transaction from ElectrumX blockchain.transaction.get RPC"); + + return HashCode.fromString((String) rawTransactionHex).asBytes(); + } + + /** + * Returns raw transaction for passed transaction hash. + *

+ * NOTE: Do not mutate returned byte[]! + * + * @throws ForeignBlockchainException.NotFoundException if transaction not found + * @throws ForeignBlockchainException if error occurs + */ + @Override + public byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException { + return getRawTransaction(HashCode.fromBytes(txHash).toString()); + } + + /** + * Returns transaction info for passed transaction hash. + *

+ * @throws ForeignBlockchainException.NotFoundException if transaction not found + * @throws ForeignBlockchainException if error occurs + */ + @Override + public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException { + // Check cache first + BitcoinyTransaction transaction = transactionCache.get(txHash); + if (transaction != null) + return transaction; + + Object transactionObj = null; + + do { + try { + transactionObj = this.rpc("blockchain.transaction.get", txHash, true); + } catch (ForeignBlockchainException.NetworkException e) { + // DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'}) + if (Integer.valueOf(-5).equals(e.getDaemonErrorCode())) + throw new ForeignBlockchainException.NotFoundException(e.getMessage()); + + // Some servers also return non-standard responses like this: + // {"error":"verbose transactions are currently unsupported","id":3,"jsonrpc":"2.0"} + // We should probably not use this server any more + if (e.getServer() != null && e.getMessage() != null && e.getMessage().contains(VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE)) { + Server uselessServer = (Server) e.getServer(); + LOGGER.trace(() -> String.format("Server %s doesn't support verbose transactions - barring use of that server", uselessServer)); + this.uselessServers.add(uselessServer); + this.closeServer(uselessServer); + continue; + } + + throw e; + } + } while (transactionObj == null); + + if (!(transactionObj instanceof JSONObject)) + throw new ForeignBlockchainException.NetworkException("Expected JSONObject as response from ElectrumX blockchain.transaction.get RPC"); + + JSONObject transactionJson = (JSONObject) transactionObj; + + Object inputsObj = transactionJson.get("vin"); + if (!(inputsObj instanceof JSONArray)) + throw new ForeignBlockchainException.NetworkException("Expected JSONArray for 'vin' from ElectrumX blockchain.transaction.get RPC"); + + Object outputsObj = transactionJson.get("vout"); + if (!(outputsObj instanceof JSONArray)) + throw new ForeignBlockchainException.NetworkException("Expected JSONArray for 'vout' from ElectrumX blockchain.transaction.get RPC"); + + try { + int size = ((Long) transactionJson.get("size")).intValue(); + int locktime = ((Long) transactionJson.get("locktime")).intValue(); + + // Timestamp might not be present, e.g. for unconfirmed transaction + Object timeObj = transactionJson.get("time"); + Integer timestamp = timeObj != null + ? ((Long) timeObj).intValue() + : null; + + List inputs = new ArrayList<>(); + for (Object inputObj : (JSONArray) inputsObj) { + JSONObject inputJson = (JSONObject) inputObj; + + String scriptSig = (String) ((JSONObject) inputJson.get("scriptSig")).get("hex"); + int sequence = ((Long) inputJson.get("sequence")).intValue(); + String outputTxHash = (String) inputJson.get("txid"); + int outputVout = ((Long) inputJson.get("vout")).intValue(); + + inputs.add(new BitcoinyTransaction.Input(scriptSig, sequence, outputTxHash, outputVout)); + } + + List outputs = new ArrayList<>(); + for (Object outputObj : (JSONArray) outputsObj) { + JSONObject outputJson = (JSONObject) outputObj; + + String scriptPubKey = (String) ((JSONObject) outputJson.get("scriptPubKey")).get("hex"); + long value = BigDecimal.valueOf((Double) outputJson.get("value")).setScale(8).unscaledValue().longValue(); + + // address too, if present + List addresses = null; + Object addressesObj = ((JSONObject) outputJson.get("scriptPubKey")).get("addresses"); + if (addressesObj instanceof JSONArray) { + addresses = new ArrayList<>(); + for (Object addressObj : (JSONArray) addressesObj) + addresses.add((String) addressObj); + } + + outputs.add(new BitcoinyTransaction.Output(scriptPubKey, value, addresses)); + } + + transaction = new BitcoinyTransaction(txHash, size, locktime, timestamp, inputs, outputs); + + // Save into cache + transactionCache.put(txHash, transaction); + + return transaction; + } catch (NullPointerException | ClassCastException e) { + // Unexpected / invalid response from ElectrumX server + } - return HashCode.fromString(rawTransactionHex).asBytes(); + throw new ForeignBlockchainException.NetworkException("Unexpected JSON format from ElectrumX blockchain.transaction.get RPC"); } - public List getAddressTransactions(byte[] script) { + /** + * Returns list of transactions, relating to passed payment script. + *

+ * @return list of related transactions, or empty list if script unknown + * @throws ForeignBlockchainException if error occurs + */ + @Override + public List getAddressTransactions(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException { byte[] scriptHash = Crypto.digest(script); Bytes.reverse(scriptHash); - JSONArray transactionsJson = (JSONArray) this.rpc("blockchain.scripthash.get_history", HashCode.fromBytes(scriptHash).toString()); - if (transactionsJson == null) - return null; + Object transactionsJson = this.rpc("blockchain.scripthash.get_history", HashCode.fromBytes(scriptHash).toString()); + if (!(transactionsJson instanceof JSONArray)) + throw new ForeignBlockchainException.NetworkException("Expected array output from ElectrumX blockchain.scripthash.get_history RPC"); - List rawTransactions = new ArrayList<>(); + List transactionHashes = new ArrayList<>(); - for (Object rawTransactionInfo : transactionsJson) { + for (Object rawTransactionInfo : (JSONArray) transactionsJson) { JSONObject transactionInfo = (JSONObject) rawTransactionInfo; - // We only want confirmed transactions - if (!transactionInfo.containsKey("height")) + Long height = (Long) transactionInfo.get("height"); + if (!includeUnconfirmed && (height == null || height == 0)) + // We only want confirmed transactions continue; String txHash = (String) transactionInfo.get("tx_hash"); - String rawTransactionHex = (String) this.rpc("blockchain.transaction.get", txHash); - if (rawTransactionHex == null) - return null; - rawTransactions.add(HashCode.fromString(rawTransactionHex).asBytes()); + transactionHashes.add(new TransactionHash(height.intValue(), txHash)); } - return rawTransactions; + return transactionHashes; } - public boolean broadcastTransaction(byte[] transactionBytes) { + /** + * Broadcasts raw transaction to network. + *

+ * @throws ForeignBlockchainException if error occurs + */ + @Override + public void broadcastTransaction(byte[] transactionBytes) throws ForeignBlockchainException { Object rawBroadcastResult = this.rpc("blockchain.transaction.broadcast", HashCode.fromBytes(transactionBytes).toString()); - if (rawBroadcastResult == null) - return false; - // If result is a String, then it is simply transaction hash. - // Otherwise result is JSON and probably contains error info instead. - return rawBroadcastResult instanceof String; + // We're expecting a simple string that is the transaction hash + if (!(rawBroadcastResult instanceof String)) + throw new ForeignBlockchainException.NetworkException("Unexpected response from ElectrumX blockchain.transaction.broadcast RPC"); } // Class-private utility methods - private Set serverPeersSubscribe() { + /** + * Query current server for its list of peer servers, and return those we can parse. + *

+ * @throws ForeignBlockchainException + * @throws ClassCastException to be handled by caller + */ + private Set serverPeersSubscribe() throws ForeignBlockchainException { Set newServers = new HashSet<>(); - JSONArray peers = (JSONArray) this.connectedRpc("server.peers.subscribe"); - if (peers == null) - return newServers; + Object peers = this.connectedRpc("server.peers.subscribe"); - for (Object rawPeer : peers) { + for (Object rawPeer : (JSONArray) peers) { JSONArray peer = (JSONArray) rawPeer; if (peer.size() < 3) + // We're expecting at least 3 fields for each peer entry: IP, hostname, features continue; String hostname = (String) peer.get(1); @@ -253,21 +498,26 @@ private Set serverPeersSubscribe() { for (Object rawFeature : features) { String feature = (String) rawFeature; Server.ConnectionType connectionType = null; - int port = -1; + Integer port = null; switch (feature.charAt(0)) { case 's': connectionType = Server.ConnectionType.SSL; - port = DEFAULT_SSL_PORT; + port = this.defaultPorts.get(connectionType); break; case 't': connectionType = Server.ConnectionType.TCP; - port = DEFAULT_TCP_PORT; + port = this.defaultPorts.get(connectionType); + break; + + default: + // e.g. could be 'v' for protocol version, or 'p' for pruning limit break; } - if (connectionType == null) + if (connectionType == null || port == null) + // We couldn't extract any peer connection info? continue; // Possible non-default port? @@ -287,32 +537,50 @@ private Set serverPeersSubscribe() { return newServers; } - private synchronized Object rpc(String method, Object...params) { - while (haveConnection()) { - Object response = connectedRpc(method, params); - if (response != null) - return response; + /** + * Performs RPC call, with automatic reconnection to different server if needed. + *

+ * @return "result" object from within JSON output + * @throws ForeignBlockchainException if server returns error or something goes wrong + */ + private Object rpc(String method, Object...params) throws ForeignBlockchainException { + synchronized (this.serverLock) { + if (this.remainingServers.isEmpty()) + this.remainingServers.addAll(this.servers); + + while (haveConnection()) { + Object response = connectedRpc(method, params); + + // If we have more servers and this one replied slowly, try another + if (!this.remainingServers.isEmpty()) { + long averageResponseTime = this.currentServer.averageResponseTime(); + if (averageResponseTime > MAX_AVG_RESPONSE_TIME) { + LOGGER.info("Slow average response time {}ms from {} - trying another server...", averageResponseTime, this.currentServer.hostname); + this.closeServer(); + break; + } + } - this.currentServer = null; - try { - this.socket.close(); - } catch (IOException e) { - /* ignore */ + if (response != null) + return response; + + // Didn't work, try another server... + this.closeServer(); } - this.scanner = null; - } - return null; + // Failed to perform RPC - maybe lack of servers? + LOGGER.info("Error: No connected Electrum servers when trying to make RPC call"); + throw new ForeignBlockchainException.NetworkException(String.format("Failed to perform ElectrumX RPC %s", method)); + } } - private boolean haveConnection() { + /** Returns true if we have, or create, a connection to an ElectrumX server. */ + private boolean haveConnection() throws ForeignBlockchainException { if (this.currentServer != null) return true; - List remainingServers = new ArrayList<>(this.servers); - - while (!remainingServers.isEmpty()) { - Server server = remainingServers.remove(RANDOM.nextInt(remainingServers.size())); + while (!this.remainingServers.isEmpty()) { + Server server = this.remainingServers.remove(RANDOM.nextInt(this.remainingServers.size())); LOGGER.trace(() -> String.format("Connecting to %s", server)); try { @@ -325,36 +593,55 @@ private boolean haveConnection() { if (server.connectionType == Server.ConnectionType.SSL) { SSLSocketFactory factory = TrustlessSSLSocketFactory.getSocketFactory(); - this.socket = (SSLSocket) factory.createSocket(this.socket, server.hostname, server.port, true); + this.socket = factory.createSocket(this.socket, server.hostname, server.port, true); } this.scanner = new Scanner(this.socket.getInputStream()); this.scanner.useDelimiter("\n"); - // Check connection works by asking for more servers + // Check connection is suitable by asking for server features, including genesis block hash + JSONObject featuresJson = (JSONObject) this.connectedRpc("server.features"); + + if (featuresJson == null || Double.valueOf((String) featuresJson.get("protocol_min")) < MIN_PROTOCOL_VERSION) + continue; + + if (this.expectedGenesisHash != null && !((String) featuresJson.get("genesis_hash")).equals(this.expectedGenesisHash)) + continue; + + // Ask for more servers Set moreServers = serverPeersSubscribe(); + // Discard duplicate servers we already know moreServers.removeAll(this.servers); - remainingServers.addAll(moreServers); + // Add to both lists + this.remainingServers.addAll(moreServers); this.servers.addAll(moreServers); LOGGER.debug(() -> String.format("Connected to %s", server)); this.currentServer = server; return true; - } catch (IOException e) { - // Try another server... - this.socket = null; - this.scanner = null; + } catch (IOException | ForeignBlockchainException | ClassCastException | NullPointerException e) { + // Didn't work, try another server... + closeServer(); } } return false; } + /** + * Perform RPC using currently connected server. + *

+ * @param method + * @param params + * @return response Object, or null if server fails to respond + * @throws ForeignBlockchainException if server returns error + */ @SuppressWarnings("unchecked") - private Object connectedRpc(String method, Object...params) { + private Object connectedRpc(String method, Object...params) throws ForeignBlockchainException { JSONObject requestJson = new JSONObject(); requestJson.put("id", this.nextId++); requestJson.put("method", method); + requestJson.put("jsonrpc", "2.0"); JSONArray requestParams = new JSONArray(); requestParams.addAll(Arrays.asList(params)); @@ -363,25 +650,110 @@ private Object connectedRpc(String method, Object...params) { String request = requestJson.toJSONString() + "\n"; LOGGER.trace(() -> String.format("Request: %s", request)); + long startTime = System.currentTimeMillis(); final String response; try { this.socket.getOutputStream().write(request.getBytes()); response = scanner.next(); } catch (IOException | NoSuchElementException e) { + // Unable to send, or receive -- try another server? return null; } + long endTime = System.currentTimeMillis(); + long responseTime = endTime-startTime; + LOGGER.trace(() -> String.format("Response: %s", response)); + LOGGER.trace(() -> String.format("Time taken: %dms", endTime-startTime)); if (response.isEmpty()) + // Empty response - try another server? return null; - JSONObject responseJson = (JSONObject) JSONValue.parse(response); - if (responseJson == null) + Object responseObj = JSONValue.parse(response); + if (!(responseObj instanceof JSONObject)) + // Unexpected response - try another server? return null; + // Keep track of response times + if (this.currentServer != null) { + this.currentServer.addResponseTime(responseTime); + } + + JSONObject responseJson = (JSONObject) responseObj; + + Object errorObj = responseJson.get("error"); + if (errorObj != null) { + if (errorObj instanceof String) { + LOGGER.debug(String.format("Unexpected error message from ElectrumX server %s for RPC method %s: %s", this.currentServer, method, (String) errorObj)); + // Try another server + return null; + } + + if (!(errorObj instanceof JSONObject)) { + LOGGER.debug(String.format("Unexpected error response from ElectrumX server %s for RPC method %s", this.currentServer, method)); + // Try another server + return null; + } + + JSONObject errorJson = (JSONObject) errorObj; + + Object messageObj = errorJson.get("message"); + + if (!(messageObj instanceof String)) { + LOGGER.debug(String.format("Missing/invalid message in error response from ElectrumX server %s for RPC method %s", this.currentServer, method)); + // Try another server + return null; + } + + String message = (String) messageObj; + + // Some error 'messages' are actually wrapped upstream bitcoind errors: + // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'})" + // We want to detect these and extract the upstream error code for caller's use + Matcher messageMatcher = DAEMON_ERROR_REGEX.matcher(message); + if (messageMatcher.find()) + try { + int daemonErrorCode = Integer.parseInt(messageMatcher.group(1)); + throw new ForeignBlockchainException.NetworkException(daemonErrorCode, message, this.currentServer); + } catch (NumberFormatException e) { + // We couldn't parse the error code integer? Fall-through to generic exception... + } + + throw new ForeignBlockchainException.NetworkException(message, this.currentServer); + } + return responseJson.get("result"); } + /** + * Closes connection to server if it is currently connected server. + * @param server + */ + private void closeServer(Server server) { + synchronized (this.serverLock) { + if (this.currentServer == null || !this.currentServer.equals(server)) + return; + + if (this.socket != null && !this.socket.isClosed()) + try { + this.socket.close(); + } catch (IOException e) { + // We did try... + } + + this.socket = null; + this.scanner = null; + this.currentServer = null; + } + } + + /** Closes connection to currently connected server (if any). */ + private void closeServer() { + synchronized (this.serverLock) { + this.closeServer(this.currentServer); + } + } + } diff --git a/src/main/java/org/qortal/crosschain/ForeignBlockchain.java b/src/main/java/org/qortal/crosschain/ForeignBlockchain.java new file mode 100644 index 000000000..fe64ab830 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/ForeignBlockchain.java @@ -0,0 +1,11 @@ +package org.qortal.crosschain; + +public interface ForeignBlockchain { + + public boolean isValidAddress(String address); + + public boolean isValidWalletKey(String walletKey); + + public long getMinimumOrderAmount(); + +} diff --git a/src/main/java/org/qortal/crosschain/ForeignBlockchainException.java b/src/main/java/org/qortal/crosschain/ForeignBlockchainException.java new file mode 100644 index 000000000..1e658621d --- /dev/null +++ b/src/main/java/org/qortal/crosschain/ForeignBlockchainException.java @@ -0,0 +1,77 @@ +package org.qortal.crosschain; + +@SuppressWarnings("serial") +public class ForeignBlockchainException extends Exception { + + public ForeignBlockchainException() { + super(); + } + + public ForeignBlockchainException(String message) { + super(message); + } + + public static class NetworkException extends ForeignBlockchainException { + private final Integer daemonErrorCode; + private final transient Object server; + + public NetworkException() { + super(); + this.daemonErrorCode = null; + this.server = null; + } + + public NetworkException(String message) { + super(message); + this.daemonErrorCode = null; + this.server = null; + } + + public NetworkException(int errorCode, String message) { + super(message); + this.daemonErrorCode = errorCode; + this.server = null; + } + + public NetworkException(String message, Object server) { + super(message); + this.daemonErrorCode = null; + this.server = server; + } + + public NetworkException(int errorCode, String message, Object server) { + super(message); + this.daemonErrorCode = errorCode; + this.server = server; + } + + public Integer getDaemonErrorCode() { + return this.daemonErrorCode; + } + + public Object getServer() { + return this.server; + } + } + + public static class NotFoundException extends ForeignBlockchainException { + public NotFoundException() { + super(); + } + + public NotFoundException(String message) { + super(message); + } + } + + public static class InsufficientFundsException extends ForeignBlockchainException { + public InsufficientFundsException() { + super(); + } + + public InsufficientFundsException(String message) { + super(message); + } + } + +} diff --git a/src/main/java/org/qortal/crosschain/Litecoin.java b/src/main/java/org/qortal/crosschain/Litecoin.java new file mode 100644 index 000000000..21ecd1db8 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/Litecoin.java @@ -0,0 +1,189 @@ +package org.qortal.crosschain; + +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumMap; +import java.util.Map; + +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.Context; +import org.bitcoinj.core.NetworkParameters; +import org.libdohj.params.LitecoinMainNetParams; +import org.libdohj.params.LitecoinRegTestParams; +import org.libdohj.params.LitecoinTestNet3Params; +import org.qortal.crosschain.ElectrumX.Server; +import org.qortal.crosschain.ElectrumX.Server.ConnectionType; +import org.qortal.settings.Settings; + +public class Litecoin extends Bitcoiny { + + public static final String CURRENCY_CODE = "LTC"; + + private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(10000); // 0.0001 LTC per 1000 bytes + + private static final long MINIMUM_ORDER_AMOUNT = 1000000; // 0.01 LTC minimum order, to avoid dust errors + + // Temporary values until a dynamic fee system is written. + private static final long MAINNET_FEE = 1000L; + private static final long NON_MAINNET_FEE = 1000L; // enough for TESTNET3 and should be OK for REGTEST + + private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ElectrumX.Server.ConnectionType.class); + static { + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.TCP, 50001); + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.SSL, 50002); + } + + public enum LitecoinNet { + MAIN { + @Override + public NetworkParameters getParams() { + return LitecoinMainNetParams.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + // Servers chosen on NO BASIS WHATSOEVER from various sources! + new Server("electrum-ltc.someguy123.net", Server.ConnectionType.SSL, 50002), + new Server("backup.electrum-ltc.org", Server.ConnectionType.TCP, 50001), + new Server("backup.electrum-ltc.org", Server.ConnectionType.SSL, 443), + new Server("electrum.ltc.xurious.com", Server.ConnectionType.TCP, 50001), + new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 50002), + new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 50002), + new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20063), + new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20063), + new Server("electrum3.cipig.net", ConnectionType.TCP, 10063), + new Server("electrum2.cipig.net", Server.ConnectionType.TCP, 10063), + new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20063), + new Server("electrum1.cipig.net", Server.ConnectionType.TCP, 10063), + new Server("electrum-ltc.petrkr.net", Server.ConnectionType.SSL, 60002), + new Server("ltc.litepay.ch", Server.ConnectionType.SSL, 50022), + new Server("electrum-ltc-bysh.me", Server.ConnectionType.TCP, 50002), + new Server("electrum.jochen-hoenicke.de", Server.ConnectionType.TCP, 50005), + new Server("node.ispol.sk", Server.ConnectionType.TCP, 50004)); + } + + @Override + public String getGenesisHash() { + return "12a765e31ffd4059bada1e25190f6e98c99d9714d334efa41a195a7e7e04bfe2"; + } + + @Override + public long getP2shFee(Long timestamp) { + // TODO: This will need to be replaced with something better in the near future! + return MAINNET_FEE; + } + }, + TEST3 { + @Override + public NetworkParameters getParams() { + return LitecoinTestNet3Params.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + new Server("electrum-ltc.bysh.me", Server.ConnectionType.TCP, 51001), + new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 51002), + new Server("electrum.ltc.xurious.com", Server.ConnectionType.TCP, 51001), + new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 51002)); + } + + @Override + public String getGenesisHash() { + return "4966625a4b2851d9fdee139e56211a0d88575f59ed816ff5e6a63deb4e3e29a0"; + } + + @Override + public long getP2shFee(Long timestamp) { + return NON_MAINNET_FEE; + } + }, + REGTEST { + @Override + public NetworkParameters getParams() { + return LitecoinRegTestParams.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + new Server("localhost", Server.ConnectionType.TCP, 50001), + new Server("localhost", Server.ConnectionType.SSL, 50002)); + } + + @Override + public String getGenesisHash() { + // This is unique to each regtest instance + return null; + } + + @Override + public long getP2shFee(Long timestamp) { + return NON_MAINNET_FEE; + } + }; + + public abstract NetworkParameters getParams(); + public abstract Collection getServers(); + public abstract String getGenesisHash(); + public abstract long getP2shFee(Long timestamp) throws ForeignBlockchainException; + } + + private static Litecoin instance; + + private final LitecoinNet litecoinNet; + + // Constructors and instance + + private Litecoin(LitecoinNet litecoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { + super(blockchain, bitcoinjContext, currencyCode); + this.litecoinNet = litecoinNet; + + LOGGER.info(() -> String.format("Starting Litecoin support using %s", this.litecoinNet.name())); + } + + public static synchronized Litecoin getInstance() { + if (instance == null) { + LitecoinNet litecoinNet = Settings.getInstance().getLitecoinNet(); + + BitcoinyBlockchainProvider electrumX = new ElectrumX("Litecoin-" + litecoinNet.name(), litecoinNet.getGenesisHash(), litecoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS); + Context bitcoinjContext = new Context(litecoinNet.getParams()); + + instance = new Litecoin(litecoinNet, electrumX, bitcoinjContext, CURRENCY_CODE); + } + + return instance; + } + + // Getters & setters + + public static synchronized void resetForTesting() { + instance = null; + } + + // Actual useful methods for use by other classes + + /** Default Litecoin fee is lower than Bitcoin: only 10sats/byte. */ + @Override + public Coin getFeePerKb() { + return DEFAULT_FEE_PER_KB; + } + + @Override + public long getMinimumOrderAmount() { + return MINIMUM_ORDER_AMOUNT; + } + + /** + * Returns estimated LTC fee, in sats per 1000bytes, optionally for historic timestamp. + * + * @param timestamp optional milliseconds since epoch, or null for 'now' + * @return sats per 1000bytes, or throws ForeignBlockchainException if something went wrong + */ + @Override + public long getP2shFee(Long timestamp) throws ForeignBlockchainException { + return this.litecoinNet.getP2shFee(timestamp); + } + +} diff --git a/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java b/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java new file mode 100644 index 000000000..efd7043e8 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java @@ -0,0 +1,854 @@ +package org.qortal.crosschain; + +import static org.ciyam.at.OpCode.calcOffset; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import org.ciyam.at.API; +import org.ciyam.at.CompilationException; +import org.ciyam.at.FunctionCode; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.ciyam.at.Timestamp; +import org.qortal.account.Account; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.utils.Base58; +import org.qortal.utils.BitTwiddling; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; + +/** + * Cross-chain trade AT + * + *

+ *

    + *
  • Bob generates Litecoin & Qortal 'trade' keys + *
      + *
    • private key required to sign P2SH redeem tx
      • + *
    • private key could be used to create 'secret' (e.g. double-SHA256)
      • + *
    • encrypted private key could be stored in Qortal AT for access by Bob from any node
      • + *
    + *
    • + *
  • Bob deploys Qortal AT + *
      + *
    + *
    • + *
  • Alice finds Qortal AT and wants to trade + *
      + *
    • Alice generates Litecoin & Qortal 'trade' keys
      • + *
    • Alice funds Litecoin P2SH-A
      • + *
    • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: + *
        + *
      • hash-of-secret-A
        • + *
      • her 'trade' Litecoin PKH
        • + *
      + *
      • + *
    + *
    • + *
  • Bob receives "offer" MESSAGE + *
      + *
    • Checks Alice's P2SH-A
      • + *
    • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: + *
        + *
      • Alice's trade Qortal address
        • + *
      • Alice's trade Litecoin PKH
        • + *
      • hash-of-secret-A
        • + *
      + *
      • + *
    + *
    • + *
  • Alice checks Qortal AT to confirm it's locked to her + *
      + *
    • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: + *
        + *
      • secret-A
        • + *
      • Qortal receiving address of her chosing
        • + *
      + *
      • + *
    • AT's QORT funds are sent to Qortal receiving address
      • + *
    + *
    • + *
  • Bob checks AT, extracts secret-A + *
      + *
    • Bob redeems P2SH-A using his Litecoin trade key and secret-A
      • + *
    • P2SH-A LTC funds end up at Litecoin address determined by redeem transaction output(s)
      • + *
    + *
    • + *
+ */ +public class LitecoinACCTv1 implements ACCT { + + public static final String NAME = LitecoinACCTv1.class.getSimpleName(); + public static final byte[] CODE_BYTES_HASH = HashCode.fromString("0fb15ad9ad1867dfbcafa51155481aa15d984ff9506f2b428eca4e2a2feac2b3").asBytes(); // SHA256 of AT code bytes + + public static final int SECRET_LENGTH = 32; + + /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ + private static final int MODE_VALUE_OFFSET = 61; + /** Byte offset into AT state data where 'mode' variable (long) is stored. */ + public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); + + public static class OfferMessageData { + public byte[] partnerLitecoinPKH; + public byte[] hashOfSecretA; + public long lockTimeA; + } + public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerLitecoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; + public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ + + 24 /*partner's Litecoin PKH (padded from 20 to 24)*/ + + 8 /*AT trade timeout (minutes)*/ + + 24 /*hash of secret-A (padded from 20 to 24)*/ + + 8 /*lockTimeA*/; + public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret-A*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; + public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; + + private static LitecoinACCTv1 instance; + + private LitecoinACCTv1() { + } + + public static synchronized LitecoinACCTv1 getInstance() { + if (instance == null) + instance = new LitecoinACCTv1(); + + return instance; + } + + @Override + public byte[] getCodeBytesHash() { + return CODE_BYTES_HASH; + } + + @Override + public int getModeByteOffset() { + return MODE_BYTE_OFFSET; + } + + @Override + public ForeignBlockchain getBlockchain() { + return Litecoin.getInstance(); + } + + /** + * Returns Qortal AT creation bytes for cross-chain trading AT. + *

+ * tradeTimeout (minutes) is the time window for the trade partner to send the + * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. + * + * @param creatorTradeAddress AT creator's trade Qortal address + * @param litecoinPublicKeyHash 20-byte HASH160 of creator's trade Litecoin public key + * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT + * @param litecoinAmount how much LTC the AT creator is expecting to trade + * @param tradeTimeout suggested timeout for entire trade + */ + public static byte[] buildQortalAT(String creatorTradeAddress, byte[] litecoinPublicKeyHash, long qortAmount, long litecoinAmount, int tradeTimeout) { + if (litecoinPublicKeyHash.length != 20) + throw new IllegalArgumentException("Litecoin public key hash should be 20 bytes"); + + // Labels for data segment addresses + int addrCounter = 0; + + // Constants (with corresponding dataByteBuffer.put*() calls below) + + final int addrCreatorTradeAddress1 = addrCounter++; + final int addrCreatorTradeAddress2 = addrCounter++; + final int addrCreatorTradeAddress3 = addrCounter++; + final int addrCreatorTradeAddress4 = addrCounter++; + + final int addrLitecoinPublicKeyHash = addrCounter; + addrCounter += 4; + + final int addrQortAmount = addrCounter++; + final int addrLitecoinAmount = addrCounter++; + final int addrTradeTimeout = addrCounter++; + + final int addrMessageTxnType = addrCounter++; + final int addrExpectedTradeMessageLength = addrCounter++; + final int addrExpectedRedeemMessageLength = addrCounter++; + + final int addrCreatorAddressPointer = addrCounter++; + final int addrQortalPartnerAddressPointer = addrCounter++; + final int addrMessageSenderPointer = addrCounter++; + + final int addrTradeMessagePartnerLitecoinPKHOffset = addrCounter++; + final int addrPartnerLitecoinPKHPointer = addrCounter++; + final int addrTradeMessageHashOfSecretAOffset = addrCounter++; + final int addrHashOfSecretAPointer = addrCounter++; + + final int addrRedeemMessageReceivingAddressOffset = addrCounter++; + + final int addrMessageDataPointer = addrCounter++; + final int addrMessageDataLength = addrCounter++; + + final int addrPartnerReceivingAddressPointer = addrCounter++; + + final int addrEndOfConstants = addrCounter; + + // Variables + + final int addrCreatorAddress1 = addrCounter++; + final int addrCreatorAddress2 = addrCounter++; + final int addrCreatorAddress3 = addrCounter++; + final int addrCreatorAddress4 = addrCounter++; + + final int addrQortalPartnerAddress1 = addrCounter++; + final int addrQortalPartnerAddress2 = addrCounter++; + final int addrQortalPartnerAddress3 = addrCounter++; + final int addrQortalPartnerAddress4 = addrCounter++; + + final int addrLockTimeA = addrCounter++; + final int addrRefundTimeout = addrCounter++; + final int addrRefundTimestamp = addrCounter++; + final int addrLastTxnTimestamp = addrCounter++; + final int addrBlockTimestamp = addrCounter++; + final int addrTxnType = addrCounter++; + final int addrResult = addrCounter++; + + final int addrMessageSender1 = addrCounter++; + final int addrMessageSender2 = addrCounter++; + final int addrMessageSender3 = addrCounter++; + final int addrMessageSender4 = addrCounter++; + + final int addrMessageLength = addrCounter++; + + final int addrMessageData = addrCounter; + addrCounter += 4; + + final int addrHashOfSecretA = addrCounter; + addrCounter += 4; + + final int addrPartnerLitecoinPKH = addrCounter; + addrCounter += 4; + + final int addrPartnerReceivingAddress = addrCounter; + addrCounter += 4; + + final int addrMode = addrCounter++; + assert addrMode == MODE_VALUE_OFFSET : String.format("addrMode %d does not match MODE_VALUE_OFFSET %d", addrMode, MODE_VALUE_OFFSET); + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // AT creator's trade Qortal address, decoded from Base58 + assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; + byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); + dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); + + // Litecoin public key hash + assert dataByteBuffer.position() == addrLitecoinPublicKeyHash * MachineState.VALUE_SIZE : "addrLitecoinPublicKeyHash incorrect"; + dataByteBuffer.put(Bytes.ensureCapacity(litecoinPublicKeyHash, 32, 0)); + + // Redeem Qort amount + assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; + dataByteBuffer.putLong(qortAmount); + + // Expected Litecoin amount + assert dataByteBuffer.position() == addrLitecoinAmount * MachineState.VALUE_SIZE : "addrLitecoinAmount incorrect"; + dataByteBuffer.putLong(litecoinAmount); + + // Suggested trade timeout (minutes) + assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; + dataByteBuffer.putLong(tradeTimeout); + + // We're only interested in MESSAGE transactions + assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; + dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); + + // Expected length of 'trade' MESSAGE data from AT creator + assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; + dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); + + // Expected length of 'redeem' MESSAGE data from trade partner + assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; + dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); + + // Index into data segment of AT creator's address, used by GET_B_IND + assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; + dataByteBuffer.putLong(addrCreatorAddress1); + + // Index into data segment of partner's Qortal address, used by SET_B_IND + assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; + dataByteBuffer.putLong(addrQortalPartnerAddress1); + + // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND + assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; + dataByteBuffer.putLong(addrMessageSender1); + + // Offset into 'trade' MESSAGE data payload for extracting partner's Litecoin PKH + assert dataByteBuffer.position() == addrTradeMessagePartnerLitecoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerLitecoinPKHOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Index into data segment of partner's Litecoin PKH, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerLitecoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerLitecoinPKHPointer incorrect"; + dataByteBuffer.putLong(addrPartnerLitecoinPKH); + + // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A + assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; + dataByteBuffer.putLong(64L); + + // Index into data segment to hash of secret A, used by GET_B_IND + assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; + dataByteBuffer.putLong(addrHashOfSecretA); + + // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address + assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Source location and length for hashing any passed secret + assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; + dataByteBuffer.putLong(addrMessageData); + assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; + dataByteBuffer.putLong(32L); + + // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; + dataByteBuffer.putLong(addrPartnerReceivingAddress); + + assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; + + // Code labels + Integer labelRefund = null; + + Integer labelTradeTxnLoop = null; + Integer labelCheckTradeTxn = null; + Integer labelCheckCancelTxn = null; + Integer labelNotTradeNorCancelTxn = null; + Integer labelCheckNonRefundTradeTxn = null; + Integer labelTradeTxnExtract = null; + Integer labelRedeemTxnLoop = null; + Integer labelCheckRedeemTxn = null; + Integer labelCheckRedeemTxnSender = null; + Integer labelPayout = null; + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); + + // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ + + /* Transaction processing loop */ + labelTradeTxnLoop = codeByteBuffer.position(); + + // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckTradeTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); + + /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + // Message sender's address matches AT creator's trade address so go process 'trade' message + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); + + /* Checking message sender for possible cancel message */ + labelCheckCancelTxn = codeByteBuffer.position(); + + // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + // Partner address is AT creator's address, so cancel offer and finish. + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + /* Not trade nor cancel message */ + labelNotTradeNorCancelTxn = codeByteBuffer.position(); + + // Loop to find another transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Possible switch-to-trade-mode message */ + labelCheckNonRefundTradeTxn = codeByteBuffer.position(); + + // Check 'trade' message we received has expected number of message bytes + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to info extraction code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); + // Message length didn't match - go back to finding another 'trade' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Extracting info from 'trade' MESSAGE transaction */ + labelTradeTxnExtract = codeByteBuffer.position(); + + // Extract message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); + + // Extract trade partner's Litecoin public key hash (PKH) from message into B + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerLitecoinPKHOffset)); + // Store partner's Litecoin PKH (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerLitecoinPKHPointer)); + // Extract AT trade timeout (minutes) (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrRefundTimeout)); + + // Grab next 32 bytes + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); + + // Extract hash-of-secret-A (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); + // Extract lockTime-A (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); + + // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); + + /* We are in 'trade mode' */ + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ + + // Fetch current block 'timestamp' + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); + // If we're not past refund 'timestamp' then look for next transaction + codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + // We're past refund 'timestamp' so go refund everything back to AT creator + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); + + /* Transaction processing loop */ + labelRedeemTxnLoop = codeByteBuffer.position(); + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckRedeemTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check message payload length */ + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to sender checking code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); + // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Check transaction's sender */ + labelCheckRedeemTxnSender = codeByteBuffer.position(); + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check 'secret-A' in transaction's message */ + + // Extract secret-A from first 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); + // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); + // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). + // Save the equality result (1 if they match, 0 otherwise) into addrResult. + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); + // If hashes don't match, addrResult will be zero so go find another transaction + codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Success! Pay arranged amount to receiving address */ + labelPayout = codeByteBuffer.position(); + + // Extract Qortal receiving address from next 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); + // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); + // Pay AT's balance to receiving address + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); + // Set redeemed mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + // Fall-through to refunding any remaining balance back to AT creator + + /* Refund balance back to AT creator */ + labelRefund = codeByteBuffer.position(); + + // Set refunded mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile LTC-QORT ACCT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + assert Arrays.equals(Crypto.digest(codeBytes), LitecoinACCTv1.CODE_BYTES_HASH) + : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { + ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + byte[] addressBytes = new byte[25]; // for general use + String atAddress = atStateData.getATAddress(); + + CrossChainTradeData tradeData = new CrossChainTradeData(); + + tradeData.foreignBlockchain = SupportedBlockchain.LITECOIN.name(); + tradeData.acctName = NAME; + + tradeData.qortalAtAddress = atAddress; + tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); + tradeData.creationTimestamp = creationTimestamp; + + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + + byte[] stateData = atStateData.getStateData(); + ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); + dataByteBuffer.position(MachineState.HEADER_LENGTH); + + /* Constants */ + + // Skip creator's trade address + dataByteBuffer.get(addressBytes); + tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Creator's Litecoin/foreign public key hash + tradeData.creatorForeignPKH = new byte[20]; + dataByteBuffer.get(tradeData.creatorForeignPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes + + // We don't use secret-B + tradeData.hashOfSecretB = null; + + // Redeem payout + tradeData.qortAmount = dataByteBuffer.getLong(); + + // Expected LTC amount + tradeData.expectedForeignAmount = dataByteBuffer.getLong(); + + // Trade timeout + tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); + + // Skip MESSAGE transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'trade' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'redeem' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Qortal trade address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message sender + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for partner's Litecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Litecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'redeem' message data offset for partner's Qortal receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message data + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip message data length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + /* End of constants / begin variables */ + + // Skip AT creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Partner's trade address (if present) + dataByteBuffer.get(addressBytes); + String qortalRecipient = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Potential lockTimeA (if in trade mode) + int lockTimeA = (int) dataByteBuffer.getLong(); + + // AT refund timeout (probably only useful for debugging) + int refundTimeout = (int) dataByteBuffer.getLong(); + + // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) + long tradeRefundTimestamp = dataByteBuffer.getLong(); + + // Skip last transaction timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip block timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary result + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message sender + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Skip message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message data + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Potential hash160 of secret A + byte[] hashOfSecretA = new byte[20]; + dataByteBuffer.get(hashOfSecretA); + dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes + + // Potential partner's Litecoin PKH + byte[] partnerLitecoinPKH = new byte[20]; + dataByteBuffer.get(partnerLitecoinPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerLitecoinPKH.length); // skip to 32 bytes + + // Partner's receiving address (if present) + byte[] partnerReceivingAddress = new byte[25]; + dataByteBuffer.get(partnerReceivingAddress); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes + + // Trade AT's 'mode' + long modeValue = dataByteBuffer.getLong(); + AcctMode mode = AcctMode.valueOf((int) (modeValue & 0xffL)); + + /* End of variables */ + + if (mode != null && mode != AcctMode.OFFERING) { + tradeData.mode = mode; + tradeData.refundTimeout = refundTimeout; + tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; + tradeData.qortalPartnerAddress = qortalRecipient; + tradeData.hashOfSecretA = hashOfSecretA; + tradeData.partnerForeignPKH = partnerLitecoinPKH; + tradeData.lockTimeA = lockTimeA; + + if (mode == AcctMode.REDEEMED) + tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); + } else { + tradeData.mode = AcctMode.OFFERING; + } + + tradeData.duplicateDeprecated(); + + return tradeData; + } + + /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ + public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); + } + + /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ + public static OfferMessageData extractOfferMessageData(byte[] messageData) { + if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) + return null; + + OfferMessageData offerMessageData = new OfferMessageData(); + offerMessageData.partnerLitecoinPKH = Arrays.copyOfRange(messageData, 0, 20); + offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); + offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); + + return offerMessageData; + } + + /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ + public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int refundTimeout) { + byte[] data = new byte[TRADE_MESSAGE_LENGTH]; + byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + byte[] refundTimeoutBytes = BitTwiddling.toBEByteArray((long) refundTimeout); + + System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); + System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); + System.arraycopy(refundTimeoutBytes, 0, data, 56, refundTimeoutBytes.length); + System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); + System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); + + return data; + } + + /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ + @Override + public byte[] buildCancelMessage(String creatorQortalAddress) { + byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; + byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); + + System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); + + return data; + } + + /** Returns 'redeem' MESSAGE payload for trade partner to send to AT. */ + public static byte[] buildRedeemMessage(byte[] secretA, String qortalReceivingAddress) { + byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; + byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); + + System.arraycopy(secretA, 0, data, 0, secretA.length); + System.arraycopy(qortalReceivingAddressBytes, 0, data, 32, qortalReceivingAddressBytes.length); + + return data; + } + + /** Returns refund timeout (minutes) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ + public static int calcRefundTimeout(long offerMessageTimestamp, int lockTimeA) { + // refund should be triggered halfway between offerMessageTimestamp and lockTimeA + return (int) ((lockTimeA - (offerMessageTimestamp / 1000L)) / 2L / 60L); + } + + @Override + public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { + String atAddress = crossChainTradeData.qortalAtAddress; + String redeemerAddress = crossChainTradeData.qortalPartnerAddress; + + // We don't have partner's public key so we check every message to AT + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); + if (messageTransactionsData == null) + return null; + + // Find 'redeem' message + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + // Check message payload type/encryption + if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) + continue; + + // Check message payload size + byte[] messageData = messageTransactionData.getData(); + if (messageData.length != REDEEM_MESSAGE_LENGTH) + // Wrong payload length + continue; + + // Check sender + if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) + // Wrong sender; + continue; + + // Extract secretA + byte[] secretA = new byte[32]; + System.arraycopy(messageData, 0, secretA, 0, secretA.length); + + byte[] hashOfSecretA = Crypto.hash160(secretA); + if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) + continue; + + return secretA; + } + + return null; + } + +} diff --git a/src/main/java/org/qortal/crosschain/LitecoinACCTv2.java b/src/main/java/org/qortal/crosschain/LitecoinACCTv2.java new file mode 100644 index 000000000..c57289534 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/LitecoinACCTv2.java @@ -0,0 +1,854 @@ +package org.qortal.crosschain; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; +import org.ciyam.at.*; +import org.qortal.account.Account; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.utils.Base58; +import org.qortal.utils.BitTwiddling; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import static org.ciyam.at.OpCode.calcOffset; + +/** + * Cross-chain trade AT + * + *

+ *

    + *
  • Bob generates Litecoin & Qortal 'trade' keys + *
      + *
    • private key required to sign P2SH redeem tx
      • + *
    • private key could be used to create 'secret' (e.g. double-SHA256)
      • + *
    • encrypted private key could be stored in Qortal AT for access by Bob from any node
      • + *
    + *
    • + *
  • Bob deploys Qortal AT + *
      + *
    + *
    • + *
  • Alice finds Qortal AT and wants to trade + *
      + *
    • Alice generates Litecoin & Qortal 'trade' keys
      • + *
    • Alice funds Litecoin P2SH-A
      • + *
    • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: + *
        + *
      • hash-of-secret-A
        • + *
      • her 'trade' Litecoin PKH
        • + *
      + *
      • + *
    + *
    • + *
  • Bob receives "offer" MESSAGE + *
      + *
    • Checks Alice's P2SH-A
      • + *
    • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: + *
        + *
      • Alice's trade Qortal address
        • + *
      • Alice's trade Litecoin PKH
        • + *
      • hash-of-secret-A
        • + *
      + *
      • + *
    + *
    • + *
  • Alice checks Qortal AT to confirm it's locked to her + *
      + *
    • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: + *
        + *
      • secret-A
        • + *
      • Qortal receiving address of her chosing
        • + *
      + *
      • + *
    • AT's QORT funds are sent to Qortal receiving address
      • + *
    + *
    • + *
  • Bob checks AT, extracts secret-A + *
      + *
    • Bob redeems P2SH-A using his Litecoin trade key and secret-A
      • + *
    • P2SH-A LTC funds end up at Litecoin address determined by redeem transaction output(s)
      • + *
    + *
    • + *
+ */ +public class LitecoinACCTv2 implements ACCT { + + public static final String NAME = LitecoinACCTv2.class.getSimpleName(); + public static final byte[] CODE_BYTES_HASH = HashCode.fromString("d5ea386a41441180c854ca8d7bbc620bfd53a97df2650a2b162b52324caf6e19").asBytes(); // SHA256 of AT code bytes + + public static final int SECRET_LENGTH = 32; + + /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ + private static final int MODE_VALUE_OFFSET = 61; + /** Byte offset into AT state data where 'mode' variable (long) is stored. */ + public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); + + public static class OfferMessageData { + public byte[] partnerLitecoinPKH; + public byte[] hashOfSecretA; + public long lockTimeA; + } + public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerLitecoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; + public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ + + 24 /*partner's Litecoin PKH (padded from 20 to 24)*/ + + 8 /*AT trade timeout (minutes)*/ + + 24 /*hash of secret-A (padded from 20 to 24)*/ + + 8 /*lockTimeA*/; + public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret-A*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; + public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; + + private static LitecoinACCTv2 instance; + + private LitecoinACCTv2() { + } + + public static synchronized LitecoinACCTv2 getInstance() { + if (instance == null) + instance = new LitecoinACCTv2(); + + return instance; + } + + @Override + public byte[] getCodeBytesHash() { + return CODE_BYTES_HASH; + } + + @Override + public int getModeByteOffset() { + return MODE_BYTE_OFFSET; + } + + @Override + public ForeignBlockchain getBlockchain() { + return Litecoin.getInstance(); + } + + /** + * Returns Qortal AT creation bytes for cross-chain trading AT. + *

+ * tradeTimeout (minutes) is the time window for the trade partner to send the + * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. + * + * @param creatorTradeAddress AT creator's trade Qortal address + * @param litecoinPublicKeyHash 20-byte HASH160 of creator's trade Litecoin public key + * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT + * @param litecoinAmount how much LTC the AT creator is expecting to trade + * @param tradeTimeout suggested timeout for entire trade + */ + public static byte[] buildQortalAT(String creatorTradeAddress, byte[] litecoinPublicKeyHash, long qortAmount, long litecoinAmount, int tradeTimeout) { + if (litecoinPublicKeyHash.length != 20) + throw new IllegalArgumentException("Litecoin public key hash should be 20 bytes"); + + // Labels for data segment addresses + int addrCounter = 0; + + // Constants (with corresponding dataByteBuffer.put*() calls below) + + final int addrCreatorTradeAddress1 = addrCounter++; + final int addrCreatorTradeAddress2 = addrCounter++; + final int addrCreatorTradeAddress3 = addrCounter++; + final int addrCreatorTradeAddress4 = addrCounter++; + + final int addrLitecoinPublicKeyHash = addrCounter; + addrCounter += 4; + + final int addrQortAmount = addrCounter++; + final int addrLitecoinAmount = addrCounter++; + final int addrTradeTimeout = addrCounter++; + + final int addrMessageTxnType = addrCounter++; + final int addrExpectedTradeMessageLength = addrCounter++; + final int addrExpectedRedeemMessageLength = addrCounter++; + + final int addrCreatorAddressPointer = addrCounter++; + final int addrQortalPartnerAddressPointer = addrCounter++; + final int addrMessageSenderPointer = addrCounter++; + + final int addrTradeMessagePartnerLitecoinPKHOffset = addrCounter++; + final int addrPartnerLitecoinPKHPointer = addrCounter++; + final int addrTradeMessageHashOfSecretAOffset = addrCounter++; + final int addrHashOfSecretAPointer = addrCounter++; + + final int addrRedeemMessageReceivingAddressOffset = addrCounter++; + + final int addrMessageDataPointer = addrCounter++; + final int addrMessageDataLength = addrCounter++; + + final int addrPartnerReceivingAddressPointer = addrCounter++; + + final int addrEndOfConstants = addrCounter; + + // Variables + + final int addrCreatorAddress1 = addrCounter++; + final int addrCreatorAddress2 = addrCounter++; + final int addrCreatorAddress3 = addrCounter++; + final int addrCreatorAddress4 = addrCounter++; + + final int addrQortalPartnerAddress1 = addrCounter++; + final int addrQortalPartnerAddress2 = addrCounter++; + final int addrQortalPartnerAddress3 = addrCounter++; + final int addrQortalPartnerAddress4 = addrCounter++; + + final int addrLockTimeA = addrCounter++; + final int addrRefundTimeout = addrCounter++; + final int addrRefundTimestamp = addrCounter++; + final int addrLastTxnTimestamp = addrCounter++; + final int addrBlockTimestamp = addrCounter++; + final int addrTxnType = addrCounter++; + final int addrResult = addrCounter++; + + final int addrMessageSender1 = addrCounter++; + final int addrMessageSender2 = addrCounter++; + final int addrMessageSender3 = addrCounter++; + final int addrMessageSender4 = addrCounter++; + + final int addrMessageLength = addrCounter++; + + final int addrMessageData = addrCounter; + addrCounter += 4; + + final int addrHashOfSecretA = addrCounter; + addrCounter += 4; + + final int addrPartnerLitecoinPKH = addrCounter; + addrCounter += 4; + + final int addrPartnerReceivingAddress = addrCounter; + addrCounter += 4; + + final int addrMode = addrCounter++; + assert addrMode == MODE_VALUE_OFFSET : String.format("addrMode %d does not match MODE_VALUE_OFFSET %d", addrMode, MODE_VALUE_OFFSET); + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // AT creator's trade Qortal address, decoded from Base58 + assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; + byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); + dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); + + // Litecoin public key hash + assert dataByteBuffer.position() == addrLitecoinPublicKeyHash * MachineState.VALUE_SIZE : "addrLitecoinPublicKeyHash incorrect"; + dataByteBuffer.put(Bytes.ensureCapacity(litecoinPublicKeyHash, 32, 0)); + + // Redeem Qort amount + assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; + dataByteBuffer.putLong(qortAmount); + + // Expected Litecoin amount + assert dataByteBuffer.position() == addrLitecoinAmount * MachineState.VALUE_SIZE : "addrLitecoinAmount incorrect"; + dataByteBuffer.putLong(litecoinAmount); + + // Suggested trade timeout (minutes) + assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; + dataByteBuffer.putLong(tradeTimeout); + + // We're only interested in MESSAGE transactions + assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; + dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); + + // Expected length of 'trade' MESSAGE data from AT creator + assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; + dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); + + // Expected length of 'redeem' MESSAGE data from trade partner + assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; + dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); + + // Index into data segment of AT creator's address, used by GET_B_IND + assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; + dataByteBuffer.putLong(addrCreatorAddress1); + + // Index into data segment of partner's Qortal address, used by SET_B_IND + assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; + dataByteBuffer.putLong(addrQortalPartnerAddress1); + + // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND + assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; + dataByteBuffer.putLong(addrMessageSender1); + + // Offset into 'trade' MESSAGE data payload for extracting partner's Litecoin PKH + assert dataByteBuffer.position() == addrTradeMessagePartnerLitecoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerLitecoinPKHOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Index into data segment of partner's Litecoin PKH, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerLitecoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerLitecoinPKHPointer incorrect"; + dataByteBuffer.putLong(addrPartnerLitecoinPKH); + + // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A + assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; + dataByteBuffer.putLong(64L); + + // Index into data segment to hash of secret A, used by GET_B_IND + assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; + dataByteBuffer.putLong(addrHashOfSecretA); + + // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address + assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Source location and length for hashing any passed secret + assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; + dataByteBuffer.putLong(addrMessageData); + assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; + dataByteBuffer.putLong(32L); + + // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; + dataByteBuffer.putLong(addrPartnerReceivingAddress); + + assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; + + // Code labels + Integer labelRefund = null; + + Integer labelTradeTxnLoop = null; + Integer labelCheckTradeTxn = null; + Integer labelCheckCancelTxn = null; + Integer labelNotTradeNorCancelTxn = null; + Integer labelCheckNonRefundTradeTxn = null; + Integer labelTradeTxnExtract = null; + Integer labelRedeemTxnLoop = null; + Integer labelCheckRedeemTxn = null; + Integer labelCheckRedeemTxnSender = null; + Integer labelPayout = null; + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); + + // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ + + /* Transaction processing loop */ + labelTradeTxnLoop = codeByteBuffer.position(); + + /* Sleep until message arrives */ + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.SLEEP_UNTIL_MESSAGE.value, addrLastTxnTimestamp)); + + // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckTradeTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); + + /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + // Message sender's address matches AT creator's trade address so go process 'trade' message + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); + + /* Checking message sender for possible cancel message */ + labelCheckCancelTxn = codeByteBuffer.position(); + + // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + // Partner address is AT creator's address, so cancel offer and finish. + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + /* Not trade nor cancel message */ + labelNotTradeNorCancelTxn = codeByteBuffer.position(); + + // Loop to find another transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Possible switch-to-trade-mode message */ + labelCheckNonRefundTradeTxn = codeByteBuffer.position(); + + // Check 'trade' message we received has expected number of message bytes + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to info extraction code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); + // Message length didn't match - go back to finding another 'trade' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Extracting info from 'trade' MESSAGE transaction */ + labelTradeTxnExtract = codeByteBuffer.position(); + + // Extract message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); + + // Extract trade partner's Litecoin public key hash (PKH) from message into B + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerLitecoinPKHOffset)); + // Store partner's Litecoin PKH (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerLitecoinPKHPointer)); + // Extract AT trade timeout (minutes) (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrRefundTimeout)); + + // Grab next 32 bytes + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); + + // Extract hash-of-secret-A (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); + // Extract lockTime-A (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); + + // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); + + /* We are in 'trade mode' */ + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ + + // Fetch current block 'timestamp' + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); + // If we're not past refund 'timestamp' then look for next transaction + codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + // We're past refund 'timestamp' so go refund everything back to AT creator + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); + + /* Transaction processing loop */ + labelRedeemTxnLoop = codeByteBuffer.position(); + + /* Sleep until message arrives */ + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.SLEEP_UNTIL_MESSAGE.value, addrLastTxnTimestamp)); + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckRedeemTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check message payload length */ + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to sender checking code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); + // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Check transaction's sender */ + labelCheckRedeemTxnSender = codeByteBuffer.position(); + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check 'secret-A' in transaction's message */ + + // Extract secret-A from first 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); + // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); + // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). + // Save the equality result (1 if they match, 0 otherwise) into addrResult. + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); + // If hashes don't match, addrResult will be zero so go find another transaction + codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Success! Pay arranged amount to receiving address */ + labelPayout = codeByteBuffer.position(); + + // Extract Qortal receiving address from next 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); + // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); + // Pay AT's balance to receiving address + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); + // Set redeemed mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + // Fall-through to refunding any remaining balance back to AT creator + + /* Refund balance back to AT creator */ + labelRefund = codeByteBuffer.position(); + + // Set refunded mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile LTC-QORT ACCT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + assert Arrays.equals(Crypto.digest(codeBytes), LitecoinACCTv2.CODE_BYTES_HASH) + : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { + ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + byte[] addressBytes = new byte[25]; // for general use + String atAddress = atStateData.getATAddress(); + + CrossChainTradeData tradeData = new CrossChainTradeData(); + + tradeData.foreignBlockchain = SupportedBlockchain.LITECOIN.name(); + tradeData.acctName = NAME; + + tradeData.qortalAtAddress = atAddress; + tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); + tradeData.creationTimestamp = creationTimestamp; + + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + + byte[] stateData = atStateData.getStateData(); + ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); + dataByteBuffer.position(MachineState.HEADER_LENGTH); + + /* Constants */ + + // Skip creator's trade address + dataByteBuffer.get(addressBytes); + tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Creator's Litecoin/foreign public key hash + tradeData.creatorForeignPKH = new byte[20]; + dataByteBuffer.get(tradeData.creatorForeignPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes + + // We don't use secret-B + tradeData.hashOfSecretB = null; + + // Redeem payout + tradeData.qortAmount = dataByteBuffer.getLong(); + + // Expected LTC amount + tradeData.expectedForeignAmount = dataByteBuffer.getLong(); + + // Trade timeout + tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); + + // Skip MESSAGE transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'trade' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'redeem' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Qortal trade address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message sender + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for partner's Litecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Litecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'redeem' message data offset for partner's Qortal receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message data + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip message data length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + /* End of constants / begin variables */ + + // Skip AT creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Partner's trade address (if present) + dataByteBuffer.get(addressBytes); + String qortalRecipient = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Potential lockTimeA (if in trade mode) + int lockTimeA = (int) dataByteBuffer.getLong(); + + // AT refund timeout (probably only useful for debugging) + int refundTimeout = (int) dataByteBuffer.getLong(); + + // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) + long tradeRefundTimestamp = dataByteBuffer.getLong(); + + // Skip last transaction timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip block timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary result + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message sender + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Skip message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message data + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Potential hash160 of secret A + byte[] hashOfSecretA = new byte[20]; + dataByteBuffer.get(hashOfSecretA); + dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes + + // Potential partner's Litecoin PKH + byte[] partnerLitecoinPKH = new byte[20]; + dataByteBuffer.get(partnerLitecoinPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerLitecoinPKH.length); // skip to 32 bytes + + // Partner's receiving address (if present) + byte[] partnerReceivingAddress = new byte[25]; + dataByteBuffer.get(partnerReceivingAddress); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes + + // Trade AT's 'mode' + long modeValue = dataByteBuffer.getLong(); + AcctMode mode = AcctMode.valueOf((int) (modeValue & 0xffL)); + + /* End of variables */ + + if (mode != null && mode != AcctMode.OFFERING) { + tradeData.mode = mode; + tradeData.refundTimeout = refundTimeout; + tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; + tradeData.qortalPartnerAddress = qortalRecipient; + tradeData.hashOfSecretA = hashOfSecretA; + tradeData.partnerForeignPKH = partnerLitecoinPKH; + tradeData.lockTimeA = lockTimeA; + + if (mode == AcctMode.REDEEMED) + tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); + } else { + tradeData.mode = AcctMode.OFFERING; + } + + tradeData.duplicateDeprecated(); + + return tradeData; + } + + /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ + public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); + } + + /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ + public static OfferMessageData extractOfferMessageData(byte[] messageData) { + if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) + return null; + + OfferMessageData offerMessageData = new OfferMessageData(); + offerMessageData.partnerLitecoinPKH = Arrays.copyOfRange(messageData, 0, 20); + offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); + offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); + + return offerMessageData; + } + + /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ + public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int refundTimeout) { + byte[] data = new byte[TRADE_MESSAGE_LENGTH]; + byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + byte[] refundTimeoutBytes = BitTwiddling.toBEByteArray((long) refundTimeout); + + System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); + System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); + System.arraycopy(refundTimeoutBytes, 0, data, 56, refundTimeoutBytes.length); + System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); + System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); + + return data; + } + + /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ + @Override + public byte[] buildCancelMessage(String creatorQortalAddress) { + byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; + byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); + + System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); + + return data; + } + + /** Returns 'redeem' MESSAGE payload for trade partner to send to AT. */ + public static byte[] buildRedeemMessage(byte[] secretA, String qortalReceivingAddress) { + byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; + byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); + + System.arraycopy(secretA, 0, data, 0, secretA.length); + System.arraycopy(qortalReceivingAddressBytes, 0, data, 32, qortalReceivingAddressBytes.length); + + return data; + } + + /** Returns refund timeout (minutes) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ + public static int calcRefundTimeout(long offerMessageTimestamp, int lockTimeA) { + // refund should be triggered halfway between offerMessageTimestamp and lockTimeA + return (int) ((lockTimeA - (offerMessageTimestamp / 1000L)) / 2L / 60L); + } + + @Override + public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { + String atAddress = crossChainTradeData.qortalAtAddress; + String redeemerAddress = crossChainTradeData.qortalPartnerAddress; + + // We don't have partner's public key so we check every message to AT + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); + if (messageTransactionsData == null) + return null; + + // Find 'redeem' message + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + // Check message payload type/encryption + if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) + continue; + + // Check message payload size + byte[] messageData = messageTransactionData.getData(); + if (messageData.length != REDEEM_MESSAGE_LENGTH) + // Wrong payload length + continue; + + // Check sender + if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) + // Wrong sender; + continue; + + // Extract secretA + byte[] secretA = new byte[32]; + System.arraycopy(messageData, 0, secretA, 0, secretA.length); + + byte[] hashOfSecretA = Crypto.hash160(secretA); + if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) + continue; + + return secretA; + } + + return null; + } + +} diff --git a/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java b/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java new file mode 100644 index 000000000..a321a7dce --- /dev/null +++ b/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java @@ -0,0 +1,851 @@ +package org.qortal.crosschain; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; +import org.ciyam.at.*; +import org.qortal.account.Account; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.utils.Base58; +import org.qortal.utils.BitTwiddling; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import static org.ciyam.at.OpCode.calcOffset; + +/** + * Cross-chain trade AT + * + *

+ *

    + *
  • Bob generates Litecoin & Qortal 'trade' keys + *
      + *
    • private key required to sign P2SH redeem tx
      • + *
    • private key could be used to create 'secret' (e.g. double-SHA256)
      • + *
    • encrypted private key could be stored in Qortal AT for access by Bob from any node
      • + *
    + *
    • + *
  • Bob deploys Qortal AT + *
      + *
    + *
    • + *
  • Alice finds Qortal AT and wants to trade + *
      + *
    • Alice generates Litecoin & Qortal 'trade' keys
      • + *
    • Alice funds Litecoin P2SH-A
      • + *
    • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: + *
        + *
      • hash-of-secret-A
        • + *
      • her 'trade' Litecoin PKH
        • + *
      + *
      • + *
    + *
    • + *
  • Bob receives "offer" MESSAGE + *
      + *
    • Checks Alice's P2SH-A
      • + *
    • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: + *
        + *
      • Alice's trade Qortal address
        • + *
      • Alice's trade Litecoin PKH
        • + *
      • hash-of-secret-A
        • + *
      + *
      • + *
    + *
    • + *
  • Alice checks Qortal AT to confirm it's locked to her + *
      + *
    • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: + *
        + *
      • secret-A
        • + *
      • Qortal receiving address of her chosing
        • + *
      + *
      • + *
    • AT's QORT funds are sent to Qortal receiving address
      • + *
    + *
    • + *
  • Bob checks AT, extracts secret-A + *
      + *
    • Bob redeems P2SH-A using his Litecoin trade key and secret-A
      • + *
    • P2SH-A LTC funds end up at Litecoin address determined by redeem transaction output(s)
      • + *
    + *
    • + *
+ */ +public class LitecoinACCTv3 implements ACCT { + + public static final String NAME = LitecoinACCTv3.class.getSimpleName(); + public static final byte[] CODE_BYTES_HASH = HashCode.fromString("31588e7ddb95a908bce310bb6dc76f011ff4693d2f8a3741fab3c989d56ce7c7").asBytes(); // SHA256 of AT code bytes + + public static final int SECRET_LENGTH = 32; + + /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ + private static final int MODE_VALUE_OFFSET = 61; + /** Byte offset into AT state data where 'mode' variable (long) is stored. */ + public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); + + public static class OfferMessageData { + public byte[] partnerLitecoinPKH; + public byte[] hashOfSecretA; + public long lockTimeA; + } + public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerLitecoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; + public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ + + 24 /*partner's Litecoin PKH (padded from 20 to 24)*/ + + 8 /*AT trade timeout (minutes)*/ + + 24 /*hash of secret-A (padded from 20 to 24)*/ + + 8 /*lockTimeA*/; + public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret-A*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; + public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; + + private static LitecoinACCTv3 instance; + + private LitecoinACCTv3() { + } + + public static synchronized LitecoinACCTv3 getInstance() { + if (instance == null) + instance = new LitecoinACCTv3(); + + return instance; + } + + @Override + public byte[] getCodeBytesHash() { + return CODE_BYTES_HASH; + } + + @Override + public int getModeByteOffset() { + return MODE_BYTE_OFFSET; + } + + @Override + public ForeignBlockchain getBlockchain() { + return Litecoin.getInstance(); + } + + /** + * Returns Qortal AT creation bytes for cross-chain trading AT. + *

+ * tradeTimeout (minutes) is the time window for the trade partner to send the + * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. + * + * @param creatorTradeAddress AT creator's trade Qortal address + * @param litecoinPublicKeyHash 20-byte HASH160 of creator's trade Litecoin public key + * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT + * @param litecoinAmount how much LTC the AT creator is expecting to trade + * @param tradeTimeout suggested timeout for entire trade + */ + public static byte[] buildQortalAT(String creatorTradeAddress, byte[] litecoinPublicKeyHash, long qortAmount, long litecoinAmount, int tradeTimeout) { + if (litecoinPublicKeyHash.length != 20) + throw new IllegalArgumentException("Litecoin public key hash should be 20 bytes"); + + // Labels for data segment addresses + int addrCounter = 0; + + // Constants (with corresponding dataByteBuffer.put*() calls below) + + final int addrCreatorTradeAddress1 = addrCounter++; + final int addrCreatorTradeAddress2 = addrCounter++; + final int addrCreatorTradeAddress3 = addrCounter++; + final int addrCreatorTradeAddress4 = addrCounter++; + + final int addrLitecoinPublicKeyHash = addrCounter; + addrCounter += 4; + + final int addrQortAmount = addrCounter++; + final int addrLitecoinAmount = addrCounter++; + final int addrTradeTimeout = addrCounter++; + + final int addrMessageTxnType = addrCounter++; + final int addrExpectedTradeMessageLength = addrCounter++; + final int addrExpectedRedeemMessageLength = addrCounter++; + + final int addrCreatorAddressPointer = addrCounter++; + final int addrQortalPartnerAddressPointer = addrCounter++; + final int addrMessageSenderPointer = addrCounter++; + + final int addrTradeMessagePartnerLitecoinPKHOffset = addrCounter++; + final int addrPartnerLitecoinPKHPointer = addrCounter++; + final int addrTradeMessageHashOfSecretAOffset = addrCounter++; + final int addrHashOfSecretAPointer = addrCounter++; + + final int addrRedeemMessageReceivingAddressOffset = addrCounter++; + + final int addrMessageDataPointer = addrCounter++; + final int addrMessageDataLength = addrCounter++; + + final int addrPartnerReceivingAddressPointer = addrCounter++; + + final int addrEndOfConstants = addrCounter; + + // Variables + + final int addrCreatorAddress1 = addrCounter++; + final int addrCreatorAddress2 = addrCounter++; + final int addrCreatorAddress3 = addrCounter++; + final int addrCreatorAddress4 = addrCounter++; + + final int addrQortalPartnerAddress1 = addrCounter++; + final int addrQortalPartnerAddress2 = addrCounter++; + final int addrQortalPartnerAddress3 = addrCounter++; + final int addrQortalPartnerAddress4 = addrCounter++; + + final int addrLockTimeA = addrCounter++; + final int addrRefundTimeout = addrCounter++; + final int addrRefundTimestamp = addrCounter++; + final int addrLastTxnTimestamp = addrCounter++; + final int addrBlockTimestamp = addrCounter++; + final int addrTxnType = addrCounter++; + final int addrResult = addrCounter++; + + final int addrMessageSender1 = addrCounter++; + final int addrMessageSender2 = addrCounter++; + final int addrMessageSender3 = addrCounter++; + final int addrMessageSender4 = addrCounter++; + + final int addrMessageLength = addrCounter++; + + final int addrMessageData = addrCounter; + addrCounter += 4; + + final int addrHashOfSecretA = addrCounter; + addrCounter += 4; + + final int addrPartnerLitecoinPKH = addrCounter; + addrCounter += 4; + + final int addrPartnerReceivingAddress = addrCounter; + addrCounter += 4; + + final int addrMode = addrCounter++; + assert addrMode == MODE_VALUE_OFFSET : String.format("addrMode %d does not match MODE_VALUE_OFFSET %d", addrMode, MODE_VALUE_OFFSET); + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // AT creator's trade Qortal address, decoded from Base58 + assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; + byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); + dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); + + // Litecoin public key hash + assert dataByteBuffer.position() == addrLitecoinPublicKeyHash * MachineState.VALUE_SIZE : "addrLitecoinPublicKeyHash incorrect"; + dataByteBuffer.put(Bytes.ensureCapacity(litecoinPublicKeyHash, 32, 0)); + + // Redeem Qort amount + assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; + dataByteBuffer.putLong(qortAmount); + + // Expected Litecoin amount + assert dataByteBuffer.position() == addrLitecoinAmount * MachineState.VALUE_SIZE : "addrLitecoinAmount incorrect"; + dataByteBuffer.putLong(litecoinAmount); + + // Suggested trade timeout (minutes) + assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; + dataByteBuffer.putLong(tradeTimeout); + + // We're only interested in MESSAGE transactions + assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; + dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); + + // Expected length of 'trade' MESSAGE data from AT creator + assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; + dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); + + // Expected length of 'redeem' MESSAGE data from trade partner + assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; + dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); + + // Index into data segment of AT creator's address, used by GET_B_IND + assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; + dataByteBuffer.putLong(addrCreatorAddress1); + + // Index into data segment of partner's Qortal address, used by SET_B_IND + assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; + dataByteBuffer.putLong(addrQortalPartnerAddress1); + + // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND + assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; + dataByteBuffer.putLong(addrMessageSender1); + + // Offset into 'trade' MESSAGE data payload for extracting partner's Litecoin PKH + assert dataByteBuffer.position() == addrTradeMessagePartnerLitecoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerLitecoinPKHOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Index into data segment of partner's Litecoin PKH, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerLitecoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerLitecoinPKHPointer incorrect"; + dataByteBuffer.putLong(addrPartnerLitecoinPKH); + + // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A + assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; + dataByteBuffer.putLong(64L); + + // Index into data segment to hash of secret A, used by GET_B_IND + assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; + dataByteBuffer.putLong(addrHashOfSecretA); + + // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address + assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Source location and length for hashing any passed secret + assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; + dataByteBuffer.putLong(addrMessageData); + assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; + dataByteBuffer.putLong(32L); + + // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; + dataByteBuffer.putLong(addrPartnerReceivingAddress); + + assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; + + // Code labels + Integer labelRefund = null; + + Integer labelTradeTxnLoop = null; + Integer labelCheckTradeTxn = null; + Integer labelCheckCancelTxn = null; + Integer labelNotTradeNorCancelTxn = null; + Integer labelCheckNonRefundTradeTxn = null; + Integer labelTradeTxnExtract = null; + Integer labelRedeemTxnLoop = null; + Integer labelCheckRedeemTxn = null; + Integer labelCheckRedeemTxnSender = null; + Integer labelPayout = null; + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); + + // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ + + /* Transaction processing loop */ + labelTradeTxnLoop = codeByteBuffer.position(); + + /* Sleep until message arrives */ + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.SLEEP_UNTIL_MESSAGE.value, addrLastTxnTimestamp)); + + // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckTradeTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); + + /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + // Message sender's address matches AT creator's trade address so go process 'trade' message + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); + + /* Checking message sender for possible cancel message */ + labelCheckCancelTxn = codeByteBuffer.position(); + + // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + // Partner address is AT creator's address, so cancel offer and finish. + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + /* Not trade nor cancel message */ + labelNotTradeNorCancelTxn = codeByteBuffer.position(); + + // Loop to find another transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Possible switch-to-trade-mode message */ + labelCheckNonRefundTradeTxn = codeByteBuffer.position(); + + // Check 'trade' message we received has expected number of message bytes + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to info extraction code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); + // Message length didn't match - go back to finding another 'trade' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Extracting info from 'trade' MESSAGE transaction */ + labelTradeTxnExtract = codeByteBuffer.position(); + + // Extract message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); + + // Extract trade partner's Litecoin public key hash (PKH) from message into B + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerLitecoinPKHOffset)); + // Store partner's Litecoin PKH (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerLitecoinPKHPointer)); + // Extract AT trade timeout (minutes) (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrRefundTimeout)); + + // Grab next 32 bytes + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); + + // Extract hash-of-secret-A (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); + // Extract lockTime-A (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); + + // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); + + /* We are in 'trade mode' */ + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ + + // Fetch current block 'timestamp' + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); + // If we're not past refund 'timestamp' then look for next transaction + codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + // We're past refund 'timestamp' so go refund everything back to AT creator + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); + + /* Transaction processing loop */ + labelRedeemTxnLoop = codeByteBuffer.position(); + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckRedeemTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check message payload length */ + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to sender checking code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); + // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Check transaction's sender */ + labelCheckRedeemTxnSender = codeByteBuffer.position(); + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check 'secret-A' in transaction's message */ + + // Extract secret-A from first 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); + // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); + // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). + // Save the equality result (1 if they match, 0 otherwise) into addrResult. + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); + // If hashes don't match, addrResult will be zero so go find another transaction + codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Success! Pay arranged amount to receiving address */ + labelPayout = codeByteBuffer.position(); + + // Extract Qortal receiving address from next 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); + // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); + // Pay AT's balance to receiving address + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); + // Set redeemed mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + // Fall-through to refunding any remaining balance back to AT creator + + /* Refund balance back to AT creator */ + labelRefund = codeByteBuffer.position(); + + // Set refunded mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile LTC-QORT ACCT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + assert Arrays.equals(Crypto.digest(codeBytes), LitecoinACCTv3.CODE_BYTES_HASH) + : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { + ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + byte[] addressBytes = new byte[25]; // for general use + String atAddress = atStateData.getATAddress(); + + CrossChainTradeData tradeData = new CrossChainTradeData(); + + tradeData.foreignBlockchain = SupportedBlockchain.LITECOIN.name(); + tradeData.acctName = NAME; + + tradeData.qortalAtAddress = atAddress; + tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); + tradeData.creationTimestamp = creationTimestamp; + + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + + byte[] stateData = atStateData.getStateData(); + ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); + dataByteBuffer.position(MachineState.HEADER_LENGTH); + + /* Constants */ + + // Skip creator's trade address + dataByteBuffer.get(addressBytes); + tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Creator's Litecoin/foreign public key hash + tradeData.creatorForeignPKH = new byte[20]; + dataByteBuffer.get(tradeData.creatorForeignPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes + + // We don't use secret-B + tradeData.hashOfSecretB = null; + + // Redeem payout + tradeData.qortAmount = dataByteBuffer.getLong(); + + // Expected LTC amount + tradeData.expectedForeignAmount = dataByteBuffer.getLong(); + + // Trade timeout + tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); + + // Skip MESSAGE transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'trade' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'redeem' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Qortal trade address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message sender + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for partner's Litecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Litecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'redeem' message data offset for partner's Qortal receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message data + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip message data length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + /* End of constants / begin variables */ + + // Skip AT creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Partner's trade address (if present) + dataByteBuffer.get(addressBytes); + String qortalRecipient = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Potential lockTimeA (if in trade mode) + int lockTimeA = (int) dataByteBuffer.getLong(); + + // AT refund timeout (probably only useful for debugging) + int refundTimeout = (int) dataByteBuffer.getLong(); + + // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) + long tradeRefundTimestamp = dataByteBuffer.getLong(); + + // Skip last transaction timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip block timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary result + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message sender + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Skip message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message data + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Potential hash160 of secret A + byte[] hashOfSecretA = new byte[20]; + dataByteBuffer.get(hashOfSecretA); + dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes + + // Potential partner's Litecoin PKH + byte[] partnerLitecoinPKH = new byte[20]; + dataByteBuffer.get(partnerLitecoinPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerLitecoinPKH.length); // skip to 32 bytes + + // Partner's receiving address (if present) + byte[] partnerReceivingAddress = new byte[25]; + dataByteBuffer.get(partnerReceivingAddress); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes + + // Trade AT's 'mode' + long modeValue = dataByteBuffer.getLong(); + AcctMode mode = AcctMode.valueOf((int) (modeValue & 0xffL)); + + /* End of variables */ + + if (mode != null && mode != AcctMode.OFFERING) { + tradeData.mode = mode; + tradeData.refundTimeout = refundTimeout; + tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; + tradeData.qortalPartnerAddress = qortalRecipient; + tradeData.hashOfSecretA = hashOfSecretA; + tradeData.partnerForeignPKH = partnerLitecoinPKH; + tradeData.lockTimeA = lockTimeA; + + if (mode == AcctMode.REDEEMED) + tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); + } else { + tradeData.mode = AcctMode.OFFERING; + } + + tradeData.duplicateDeprecated(); + + return tradeData; + } + + /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ + public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); + } + + /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ + public static OfferMessageData extractOfferMessageData(byte[] messageData) { + if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) + return null; + + OfferMessageData offerMessageData = new OfferMessageData(); + offerMessageData.partnerLitecoinPKH = Arrays.copyOfRange(messageData, 0, 20); + offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); + offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); + + return offerMessageData; + } + + /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ + public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int refundTimeout) { + byte[] data = new byte[TRADE_MESSAGE_LENGTH]; + byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + byte[] refundTimeoutBytes = BitTwiddling.toBEByteArray((long) refundTimeout); + + System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); + System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); + System.arraycopy(refundTimeoutBytes, 0, data, 56, refundTimeoutBytes.length); + System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); + System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); + + return data; + } + + /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ + @Override + public byte[] buildCancelMessage(String creatorQortalAddress) { + byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; + byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); + + System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); + + return data; + } + + /** Returns 'redeem' MESSAGE payload for trade partner to send to AT. */ + public static byte[] buildRedeemMessage(byte[] secretA, String qortalReceivingAddress) { + byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; + byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); + + System.arraycopy(secretA, 0, data, 0, secretA.length); + System.arraycopy(qortalReceivingAddressBytes, 0, data, 32, qortalReceivingAddressBytes.length); + + return data; + } + + /** Returns refund timeout (minutes) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ + public static int calcRefundTimeout(long offerMessageTimestamp, int lockTimeA) { + // refund should be triggered halfway between offerMessageTimestamp and lockTimeA + return (int) ((lockTimeA - (offerMessageTimestamp / 1000L)) / 2L / 60L); + } + + @Override + public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { + String atAddress = crossChainTradeData.qortalAtAddress; + String redeemerAddress = crossChainTradeData.qortalPartnerAddress; + + // We don't have partner's public key so we check every message to AT + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); + if (messageTransactionsData == null) + return null; + + // Find 'redeem' message + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + // Check message payload type/encryption + if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) + continue; + + // Check message payload size + byte[] messageData = messageTransactionData.getData(); + if (messageData.length != REDEEM_MESSAGE_LENGTH) + // Wrong payload length + continue; + + // Check sender + if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) + // Wrong sender; + continue; + + // Extract secretA + byte[] secretA = new byte[32]; + System.arraycopy(messageData, 0, secretA, 0, secretA.length); + + byte[] hashOfSecretA = Crypto.hash160(secretA); + if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) + continue; + + return secretA; + } + + return null; + } + +} diff --git a/src/main/java/org/qortal/crosschain/SimpleTransaction.java b/src/main/java/org/qortal/crosschain/SimpleTransaction.java new file mode 100644 index 000000000..27c9f9e39 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/SimpleTransaction.java @@ -0,0 +1,109 @@ +package org.qortal.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.List; + +@XmlAccessorType(XmlAccessType.FIELD) +public class SimpleTransaction { + private String txHash; + private Integer timestamp; + private long totalAmount; + private long feeAmount; + private List inputs; + private List outputs; + + + @XmlAccessorType(XmlAccessType.FIELD) + public static class Input { + private String address; + private long amount; + private boolean addressInWallet; + + public Input() { + } + + public Input(String address, long amount, boolean addressInWallet) { + this.address = address; + this.amount = amount; + this.addressInWallet = addressInWallet; + } + + public String getAddress() { + return address; + } + + public long getAmount() { + return amount; + } + + public boolean getAddressInWallet() { + return addressInWallet; + } + } + + @XmlAccessorType(XmlAccessType.FIELD) + public static class Output { + private String address; + private long amount; + private boolean addressInWallet; + + public Output() { + } + + public Output(String address, long amount, boolean addressInWallet) { + this.address = address; + this.amount = amount; + this.addressInWallet = addressInWallet; + } + + public String getAddress() { + return address; + } + + public long getAmount() { + return amount; + } + + public boolean getAddressInWallet() { + return addressInWallet; + } + } + + + public SimpleTransaction() { + } + + public SimpleTransaction(String txHash, Integer timestamp, long totalAmount, long feeAmount, List inputs, List outputs) { + this.txHash = txHash; + this.timestamp = timestamp; + this.totalAmount = totalAmount; + this.feeAmount = feeAmount; + this.inputs = inputs; + this.outputs = outputs; + } + + public String getTxHash() { + return txHash; + } + + public Integer getTimestamp() { + return timestamp; + } + + public long getTotalAmount() { + return totalAmount; + } + + public long getFeeAmount() { + return feeAmount; + } + + public List getInputs() { + return this.inputs; + } + + public List getOutputs() { + return this.outputs; + } +} diff --git a/src/main/java/org/qortal/crosschain/SupportedBlockchain.java b/src/main/java/org/qortal/crosschain/SupportedBlockchain.java new file mode 100644 index 000000000..5bff7ac9b --- /dev/null +++ b/src/main/java/org/qortal/crosschain/SupportedBlockchain.java @@ -0,0 +1,131 @@ +package org.qortal.crosschain; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.qortal.utils.ByteArray; +import org.qortal.utils.Triple; + +public enum SupportedBlockchain { + + BITCOIN(Arrays.asList( + Triple.valueOf(BitcoinACCTv1.NAME, BitcoinACCTv1.CODE_BYTES_HASH, BitcoinACCTv1::getInstance) + // Could add improved BitcoinACCTv2 here in the future + )) { + @Override + public ForeignBlockchain getInstance() { + return Bitcoin.getInstance(); + } + + @Override + public ACCT getLatestAcct() { + return BitcoinACCTv1.getInstance(); + } + }, + + LITECOIN(Arrays.asList( + Triple.valueOf(LitecoinACCTv1.NAME, LitecoinACCTv1.CODE_BYTES_HASH, LitecoinACCTv1::getInstance), + Triple.valueOf(LitecoinACCTv2.NAME, LitecoinACCTv2.CODE_BYTES_HASH, LitecoinACCTv2::getInstance), + Triple.valueOf(LitecoinACCTv3.NAME, LitecoinACCTv3.CODE_BYTES_HASH, LitecoinACCTv3::getInstance) + )) { + @Override + public ForeignBlockchain getInstance() { + return Litecoin.getInstance(); + } + + @Override + public ACCT getLatestAcct() { + return LitecoinACCTv3.getInstance(); + } + }, + + DOGECOIN(Arrays.asList( + Triple.valueOf(DogecoinACCTv1.NAME, DogecoinACCTv1.CODE_BYTES_HASH, DogecoinACCTv1::getInstance), + Triple.valueOf(DogecoinACCTv2.NAME, DogecoinACCTv2.CODE_BYTES_HASH, DogecoinACCTv2::getInstance), + Triple.valueOf(DogecoinACCTv3.NAME, DogecoinACCTv3.CODE_BYTES_HASH, DogecoinACCTv3::getInstance) + )) { + @Override + public ForeignBlockchain getInstance() { + return Dogecoin.getInstance(); + } + + @Override + public ACCT getLatestAcct() { + return DogecoinACCTv3.getInstance(); + } + }; + + private static final Map> supportedAcctsByCodeHash = Arrays.stream(SupportedBlockchain.values()) + .map(supportedBlockchain -> supportedBlockchain.supportedAccts) + .flatMap(List::stream) + .collect(Collectors.toUnmodifiableMap(triple -> new ByteArray(triple.getB()), Triple::getC)); + + private static final Map> supportedAcctsByName = Arrays.stream(SupportedBlockchain.values()) + .map(supportedBlockchain -> supportedBlockchain.supportedAccts) + .flatMap(List::stream) + .collect(Collectors.toUnmodifiableMap(Triple::getA, Triple::getC)); + + private static final Map blockchainsByName = Arrays.stream(SupportedBlockchain.values()) + .collect(Collectors.toUnmodifiableMap(Enum::name, blockchain -> blockchain)); + + private final List>> supportedAccts; + + SupportedBlockchain(List>> supportedAccts) { + this.supportedAccts = supportedAccts; + } + + public abstract ForeignBlockchain getInstance(); + public abstract ACCT getLatestAcct(); + + public static Map> getAcctMap() { + return supportedAcctsByCodeHash; + } + + public static SupportedBlockchain fromString(String name) { + return blockchainsByName.get(name); + } + + public static Map> getFilteredAcctMap(SupportedBlockchain blockchain) { + if (blockchain == null) + return getAcctMap(); + + return blockchain.supportedAccts.stream() + .collect(Collectors.toUnmodifiableMap(triple -> new ByteArray(triple.getB()), Triple::getC)); + } + + public static Map> getFilteredAcctMap(String specificBlockchain) { + if (specificBlockchain == null) + return getAcctMap(); + + SupportedBlockchain blockchain = blockchainsByName.get(specificBlockchain); + if (blockchain == null) + return Collections.emptyMap(); + + return getFilteredAcctMap(blockchain); + } + + public static ACCT getAcctByCodeHash(byte[] codeHash) { + ByteArray wrappedCodeHash = new ByteArray(codeHash); + + Supplier acctInstanceSupplier = supportedAcctsByCodeHash.get(wrappedCodeHash); + + if (acctInstanceSupplier == null) + return null; + + return acctInstanceSupplier.get(); + } + + public static ACCT getAcctByName(String acctName) { + Supplier acctInstanceSupplier = supportedAcctsByName.get(acctName); + + if (acctInstanceSupplier == null) + return null; + + return acctInstanceSupplier.get(); + } + +} diff --git a/src/main/java/org/qortal/crosschain/TransactionHash.java b/src/main/java/org/qortal/crosschain/TransactionHash.java new file mode 100644 index 000000000..c002ae80f --- /dev/null +++ b/src/main/java/org/qortal/crosschain/TransactionHash.java @@ -0,0 +1,31 @@ +package org.qortal.crosschain; + +import java.util.Comparator; + +public class TransactionHash { + + public static final Comparator CONFIRMED_FIRST = (a, b) -> Boolean.compare(a.height != 0, b.height != 0); + + public final int height; + public final String txHash; + + public TransactionHash(int height, String txHash) { + this.height = height; + this.txHash = txHash; + } + + public int getHeight() { + return this.height; + } + + public String getTxHash() { + return this.txHash; + } + + public String toString() { + return this.height == 0 + ? String.format("txHash %s (unconfirmed)", this.txHash) + : String.format("txHash %s (height %d)", this.txHash, this.height); + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/UnspentOutput.java b/src/main/java/org/qortal/crosschain/UnspentOutput.java new file mode 100644 index 000000000..86aa533d9 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/UnspentOutput.java @@ -0,0 +1,16 @@ +package org.qortal.crosschain; + +/** Unspent output info as returned by ElectrumX network. */ +public class UnspentOutput { + public final byte[] hash; + public final int index; + public final int height; + public final long value; + + public UnspentOutput(byte[] hash, int index, int height, long value) { + this.hash = hash; + this.index = index; + this.height = height; + this.value = value; + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/crypto/AES.java b/src/main/java/org/qortal/crypto/AES.java new file mode 100644 index 000000000..0e8018f5d --- /dev/null +++ b/src/main/java/org/qortal/crypto/AES.java @@ -0,0 +1,205 @@ +/* + * MIT License + * + * Copyright (c) 2017 Eugen Paraschiv + * Modified in 2021 by CalDescent + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +package org.qortal.crypto; + +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.BadPaddingException; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKeyFactory; +import javax.crypto.SealedObject; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.Serializable; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.KeySpec; +import java.util.Base64; + +public class AES { + + public static String encrypt(String algorithm, String input, SecretKey key, IvParameterSpec iv) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, + InvalidKeyException, BadPaddingException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.ENCRYPT_MODE, key, iv); + byte[] cipherText = cipher.doFinal(input.getBytes()); + return Base64.getEncoder() + .encodeToString(cipherText); + } + + public static String decrypt(String algorithm, String cipherText, SecretKey key, IvParameterSpec iv) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, + InvalidKeyException, BadPaddingException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.DECRYPT_MODE, key, iv); + byte[] plainText = cipher.doFinal(Base64.getDecoder() + .decode(cipherText)); + return new String(plainText); + } + + public static SecretKey generateKey(int n) throws NoSuchAlgorithmException { + KeyGenerator keyGenerator = KeyGenerator.getInstance("AES"); + keyGenerator.init(n); + SecretKey key = keyGenerator.generateKey(); + return key; + } + + public static SecretKey getKeyFromPassword(String password, String salt) + throws NoSuchAlgorithmException, InvalidKeySpecException { + SecretKeyFactory factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256"); + KeySpec spec = new PBEKeySpec(password.toCharArray(), salt.getBytes(), 65536, 256); + SecretKey secret = new SecretKeySpec(factory.generateSecret(spec) + .getEncoded(), "AES"); + return secret; + } + + public static IvParameterSpec generateIv() { + byte[] iv = new byte[16]; + new SecureRandom().nextBytes(iv); + return new IvParameterSpec(iv); + } + + public static void encryptFile(String algorithm, SecretKey key, + String inputFilePath, String outputFilePath) throws IOException, + NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException, + BadPaddingException, IllegalBlockSizeException { + + File inputFile = new File(inputFilePath); + File outputFile = new File(outputFilePath); + + IvParameterSpec iv = AES.generateIv(); + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.ENCRYPT_MODE, key, iv); + FileInputStream inputStream = new FileInputStream(inputFile); + FileOutputStream outputStream = new FileOutputStream(outputFile); + + // Prepend the output stream with the 16 byte initialization vector + outputStream.write(iv.getIV()); + + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + byte[] output = cipher.update(buffer, 0, bytesRead); + if (output != null) { + outputStream.write(output); + } + } + byte[] outputBytes = cipher.doFinal(); + if (outputBytes != null) { + outputStream.write(outputBytes); + } + inputStream.close(); + outputStream.close(); + } + + public static void decryptFile(String algorithm, SecretKey key, String encryptedFilePath, + String decryptedFilePath) throws IOException, NoSuchPaddingException, + NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException, + BadPaddingException, IllegalBlockSizeException { + + File encryptedFile = new File(encryptedFilePath); + File decryptedFile = new File(decryptedFilePath); + + File parent = decryptedFile.getParentFile(); + if (!parent.isDirectory() && !parent.mkdirs()) { + throw new IOException("Failed to create directory " + parent); + } + + FileInputStream inputStream = new FileInputStream(encryptedFile); + FileOutputStream outputStream = new FileOutputStream(decryptedFile); + + // Read the initialization vector from the first 16 bytes of the file + byte[] iv = new byte[16]; + inputStream.read(iv); + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv)); + + byte[] buffer = new byte[64]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + byte[] output = cipher.update(buffer, 0, bytesRead); + if (output != null) { + outputStream.write(output); + } + } + byte[] output = cipher.doFinal(); + if (output != null) { + outputStream.write(output); + } + inputStream.close(); + outputStream.close(); + } + + public static SealedObject encryptObject(String algorithm, Serializable object, SecretKey key, + IvParameterSpec iv) throws NoSuchPaddingException, NoSuchAlgorithmException, + InvalidAlgorithmParameterException, InvalidKeyException, IOException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.ENCRYPT_MODE, key, iv); + SealedObject sealedObject = new SealedObject(object, cipher); + return sealedObject; + } + + public static Serializable decryptObject(String algorithm, SealedObject sealedObject, SecretKey key, + IvParameterSpec iv) throws NoSuchPaddingException, NoSuchAlgorithmException, + InvalidAlgorithmParameterException, InvalidKeyException, ClassNotFoundException, + BadPaddingException, IllegalBlockSizeException, IOException { + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.DECRYPT_MODE, key, iv); + Serializable unsealObject = (Serializable) sealedObject.getObject(cipher); + return unsealObject; + } + + public static String encryptPasswordBased(String plainText, SecretKey key, IvParameterSpec iv) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, + InvalidKeyException, BadPaddingException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); + cipher.init(Cipher.ENCRYPT_MODE, key, iv); + return Base64.getEncoder() + .encodeToString(cipher.doFinal(plainText.getBytes())); + } + + public static String decryptPasswordBased(String cipherText, SecretKey key, IvParameterSpec iv) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, + InvalidKeyException, BadPaddingException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5PADDING"); + cipher.init(Cipher.DECRYPT_MODE, key, iv); + return new String(cipher.doFinal(Base64.getDecoder() + .decode(cipherText))); + } + +} diff --git a/src/main/java/org/qortal/crypto/Crypto.java b/src/main/java/org/qortal/crypto/Crypto.java index a21ac5949..5d91781c6 100644 --- a/src/main/java/org/qortal/crypto/Crypto.java +++ b/src/main/java/org/qortal/crypto/Crypto.java @@ -1,5 +1,9 @@ package org.qortal.crypto; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Arrays; @@ -42,6 +46,27 @@ public static byte[] digest(byte[] input) { } } + /** + * Returns 32-byte SHA-256 digest of message passed in input. + * + * @param input + * variable-length byte[] message + * @return byte[32] digest, or null if SHA-256 algorithm can't be accessed + */ + public static byte[] digest(ByteBuffer input) { + if (input == null) + return null; + + try { + // SHA2-256 + MessageDigest sha256 = MessageDigest.getInstance("SHA-256"); + sha256.update(input); + return sha256.digest(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("SHA-256 message digest not available"); + } + } + /** * Returns 32-byte digest of two rounds of SHA-256 on message passed in input. * @@ -53,12 +78,74 @@ public static byte[] doubleDigest(byte[] input) { return digest(digest(input)); } + /** + * Returns 32-byte SHA-256 digest of file passed in input. + * + * @param file + * file in which to perform digest + * @return byte[32] digest, or null if SHA-256 algorithm can't be accessed + * + * @throws IOException if the file cannot be read + */ + public static byte[] digest(File file) throws IOException { + return Crypto.digest(file, 8192); + } + + /** + * Returns 32-byte SHA-256 digest of file passed in input, in hex format + * + * @param file + * file in which to perform digest + * @return String digest as a hexadecimal string, or null if SHA-256 algorithm can't be accessed + * + * @throws IOException if the file cannot be read + */ + public static String digestHexString(File file, int bufferSize) throws IOException { + byte[] digest = Crypto.digest(file, bufferSize); + + // Convert to hex + StringBuilder stringBuilder = new StringBuilder(); + for (byte b : digest) { + stringBuilder.append(String.format("%02x", b)); + } + return stringBuilder.toString(); + } + + /** + * Returns 32-byte SHA-256 digest of file passed in input. + * + * @param file + * file in which to perform digest + * @param bufferSize + * the number of bytes to load into memory + * @return byte[32] digest, or null if SHA-256 algorithm can't be accessed + * + * @throws IOException if the file cannot be read + */ + public static byte[] digest(File file, int bufferSize) throws IOException { + try { + MessageDigest sha256 = MessageDigest.getInstance("SHA-256"); + FileInputStream fileInputStream = new FileInputStream(file); + byte[] bytes = new byte[bufferSize]; + int count; + + while ((count = fileInputStream.read(bytes)) != -1) { + sha256.update(bytes, 0, count); + } + fileInputStream.close(); + + return sha256.digest(); + + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("SHA-256 message digest not available"); + } + } + /** * Returns 64-byte duplicated digest of message passed in input. *

* Effectively Bytes.concat(digest(input), digest(input)). - * - * @param addressVersion + * * @param input */ public static byte[] dupDigest(byte[] input) { diff --git a/src/main/java/org/qortal/crypto/MemoryPoW.java b/src/main/java/org/qortal/crypto/MemoryPoW.java index 82e88248a..01f4f6fd8 100644 --- a/src/main/java/org/qortal/crypto/MemoryPoW.java +++ b/src/main/java/org/qortal/crypto/MemoryPoW.java @@ -29,6 +29,10 @@ public static Integer compute2(byte[] data, int workBufferLength, long difficult do { ++nonce; + // If we've been interrupted, exit fast with invalid value + if (Thread.currentThread().isInterrupted()) + return -1; + seed *= seedMultiplier; // per nonce state[0] = longHash[0] ^ seed; diff --git a/src/main/java/org/qortal/data/account/EligibleQoraHolderData.java b/src/main/java/org/qortal/data/account/EligibleQoraHolderData.java new file mode 100644 index 000000000..f3f02862e --- /dev/null +++ b/src/main/java/org/qortal/data/account/EligibleQoraHolderData.java @@ -0,0 +1,48 @@ +package org.qortal.data.account; + +public class EligibleQoraHolderData { + + // Properties + + private String address; + + private long qoraBalance; + private long qortFromQoraBalance; + + private Long finalQortFromQora; + private Integer finalBlockHeight; + + // Constructors + + public EligibleQoraHolderData(String address, long qoraBalance, long qortFromQoraBalance, Long finalQortFromQora, + Integer finalBlockHeight) { + this.address = address; + this.qoraBalance = qoraBalance; + this.qortFromQoraBalance = qortFromQoraBalance; + this.finalQortFromQora = finalQortFromQora; + this.finalBlockHeight = finalBlockHeight; + } + + // Getters/Setters + + public String getAddress() { + return this.address; + } + + public long getQoraBalance() { + return this.qoraBalance; + } + + public long getQortFromQoraBalance() { + return this.qortFromQoraBalance; + } + + public Long getFinalQortFromQora() { + return this.finalQortFromQora; + } + + public Integer getFinalBlockHeight() { + return this.finalBlockHeight; + } + +} diff --git a/src/main/java/org/qortal/data/account/MintingAccountData.java b/src/main/java/org/qortal/data/account/MintingAccountData.java index 02b4c0f8a..63c6c723a 100644 --- a/src/main/java/org/qortal/data/account/MintingAccountData.java +++ b/src/main/java/org/qortal/data/account/MintingAccountData.java @@ -4,10 +4,12 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlTransient; +import org.json.JSONObject; import org.qortal.crypto.Crypto; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.media.Schema.AccessMode; +import org.qortal.utils.Base58; // All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) @@ -61,4 +63,21 @@ public byte[] getPublicKey() { return this.publicKey; } + + // JSON + + public JSONObject toJson() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("privateKey", Base58.encode(this.getPrivateKey())); + jsonObject.put("publicKey", Base58.encode(this.getPublicKey())); + return jsonObject; + } + + public static MintingAccountData fromJson(JSONObject json) { + return new MintingAccountData( + json.isNull("privateKey") ? null : Base58.decode(json.getString("privateKey")), + json.isNull("publicKey") ? null : Base58.decode(json.getString("publicKey")) + ); + } + } diff --git a/src/main/java/org/qortal/data/account/RewardShareData.java b/src/main/java/org/qortal/data/account/RewardShareData.java index c68e42571..ead1bbd92 100644 --- a/src/main/java/org/qortal/data/account/RewardShareData.java +++ b/src/main/java/org/qortal/data/account/RewardShareData.java @@ -1,11 +1,15 @@ package org.qortal.data.account; +import java.math.BigDecimal; + import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import org.qortal.utils.Base58; + import io.swagger.v3.oas.annotations.media.Schema; // All properties to be converted to JSON via JAXB @@ -71,4 +75,13 @@ public String getMintingAccount() { return this.minter; } + // For debugging + + public String toString() { + if (this.minter.equals(this.recipient)) + return String.format("Minter/recipient: %s, reward-share public key: %s", this.minter, Base58.encode(this.rewardSharePublicKey)); + else + return String.format("Minter: %s, recipient: %s (%s %%), reward-share public key: %s", this.minter, this.recipient, BigDecimal.valueOf(this.sharePercent, 2), Base58.encode(this.rewardSharePublicKey)); + } + } diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java new file mode 100644 index 000000000..94f41d182 --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java @@ -0,0 +1,60 @@ +package org.qortal.data.arbitrary; + +import org.qortal.network.Peer; +import java.util.Objects; + +public class ArbitraryRelayInfo { + + private final String hash58; + private final String signature58; + private final Peer peer; + private final Long timestamp; + + public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp) { + this.hash58 = hash58; + this.signature58 = signature58; + this.peer = peer; + this.timestamp = timestamp; + } + + public boolean isValid() { + return this.getHash58() != null && this.getSignature58() != null + && this.getPeer() != null && this.getTimestamp() != null; + } + + public String getHash58() { + return this.hash58; + } + + public String getSignature58() { + return signature58; + } + + public Peer getPeer() { + return peer; + } + + public Long getTimestamp() { + return timestamp; + } + + @Override + public String toString() { + return String.format("%s = %s, %s, %d", this.hash58, this.signature58, this.peer, this.timestamp); + } + + @Override + public boolean equals(Object other) { + if (other == this) + return true; + + if (!(other instanceof ArbitraryRelayInfo)) + return false; + + ArbitraryRelayInfo otherRelayInfo = (ArbitraryRelayInfo) other; + + return this.peer == otherRelayInfo.getPeer() + && Objects.equals(this.hash58, otherRelayInfo.getHash58()) + && Objects.equals(this.signature58, otherRelayInfo.getSignature58()); + } +} diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java new file mode 100644 index 000000000..d3605d65d --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java @@ -0,0 +1,37 @@ +package org.qortal.data.arbitrary; + +import org.qortal.arbitrary.misc.Service; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ArbitraryResourceInfo { + + public String name; + public Service service; + public String identifier; + public ArbitraryResourceStatus status; + + public Long size; + + public ArbitraryResourceInfo() { + } + + @Override + public boolean equals(Object o) { + if (o == this) + return true; + + if (!(o instanceof ArbitraryResourceInfo)) + return false; + + ArbitraryResourceInfo other = (ArbitraryResourceInfo) o; + + return Objects.equals(this.name, other.name) && + Objects.equals(this.service, other.service) && + Objects.equals(this.identifier, other.identifier); + } + +} diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java new file mode 100644 index 000000000..b9be80341 --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java @@ -0,0 +1,17 @@ +package org.qortal.data.arbitrary; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.ArrayList; +import java.util.List; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ArbitraryResourceNameInfo { + + public String name; + public List resources = new ArrayList<>(); + + public ArbitraryResourceNameInfo() { + } + +} diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java new file mode 100644 index 000000000..5e6ac0554 --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java @@ -0,0 +1,50 @@ +package org.qortal.data.arbitrary; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ArbitraryResourceStatus { + + public enum Status { + PUBLISHED("Published", "Published but not yet downloaded"), + DOWNLOADING("Downloading", "Locating and downloading files..."), + DOWNLOADED("Downloaded", "Files downloaded"), + BUILDING("Building", "Building..."), + READY("Ready", "Ready"), + MISSING_DATA("Missing data", "Unable to locate all files. Please try again later"), + BUILD_FAILED("Build failed", "Build failed. Please try again later"), + UNSUPPORTED("Unsupported", "Unsupported request"), + BLOCKED("Blocked", "Name is blocked so content cannot be served"); + + private String title; + private String description; + + Status(String title, String description) { + this.title = title; + this.description = description; + } + } + + private String id; + private String title; + private String description; + + private Integer localChunkCount; + private Integer totalChunkCount; + + public ArbitraryResourceStatus() { + } + + public ArbitraryResourceStatus(Status status, Integer localChunkCount, Integer totalChunkCount) { + this.id = status.toString(); + this.title = status.title; + this.description = status.description; + this.localChunkCount = localChunkCount; + this.totalChunkCount = totalChunkCount; + } + + public ArbitraryResourceStatus(Status status) { + this(status, null, null); + } +} diff --git a/src/main/java/org/qortal/data/at/ATData.java b/src/main/java/org/qortal/data/at/ATData.java index 02f79f84a..9e977acf2 100644 --- a/src/main/java/org/qortal/data/at/ATData.java +++ b/src/main/java/org/qortal/data/at/ATData.java @@ -23,6 +23,7 @@ public class ATData { private boolean isFrozen; @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) private Long frozenBalance; + private Long sleepUntilMessageTimestamp; // Constructors @@ -31,7 +32,8 @@ protected ATData() { } public ATData(String ATAddress, byte[] creatorPublicKey, long creation, int version, long assetId, byte[] codeBytes, byte[] codeHash, - boolean isSleeping, Integer sleepUntilHeight, boolean isFinished, boolean hadFatalError, boolean isFrozen, Long frozenBalance) { + boolean isSleeping, Integer sleepUntilHeight, boolean isFinished, boolean hadFatalError, boolean isFrozen, Long frozenBalance, + Long sleepUntilMessageTimestamp) { this.ATAddress = ATAddress; this.creatorPublicKey = creatorPublicKey; this.creation = creation; @@ -45,6 +47,7 @@ public ATData(String ATAddress, byte[] creatorPublicKey, long creation, int vers this.hadFatalError = hadFatalError; this.isFrozen = isFrozen; this.frozenBalance = frozenBalance; + this.sleepUntilMessageTimestamp = sleepUntilMessageTimestamp; } /** For constructing skeleton ATData with bare minimum info. */ @@ -133,4 +136,12 @@ public void setFrozenBalance(Long frozenBalance) { this.frozenBalance = frozenBalance; } + public Long getSleepUntilMessageTimestamp() { + return this.sleepUntilMessageTimestamp; + } + + public void setSleepUntilMessageTimestamp(Long sleepUntilMessageTimestamp) { + this.sleepUntilMessageTimestamp = sleepUntilMessageTimestamp; + } + } diff --git a/src/main/java/org/qortal/data/at/ATStateData.java b/src/main/java/org/qortal/data/at/ATStateData.java index b8c13e0d7..ddace8e35 100644 --- a/src/main/java/org/qortal/data/at/ATStateData.java +++ b/src/main/java/org/qortal/data/at/ATStateData.java @@ -5,42 +5,37 @@ public class ATStateData { // Properties private String ATAddress; private Integer height; - private Long creation; private byte[] stateData; private byte[] stateHash; private Long fees; private boolean isInitial; + // Qortal-AT-specific + private Long sleepUntilMessageTimestamp; + // Constructors /** Create new ATStateData */ - public ATStateData(String ATAddress, Integer height, Long creation, byte[] stateData, byte[] stateHash, Long fees, boolean isInitial) { + public ATStateData(String ATAddress, Integer height, byte[] stateData, byte[] stateHash, Long fees, + boolean isInitial, Long sleepUntilMessageTimestamp) { this.ATAddress = ATAddress; this.height = height; - this.creation = creation; this.stateData = stateData; this.stateHash = stateHash; this.fees = fees; this.isInitial = isInitial; + this.sleepUntilMessageTimestamp = sleepUntilMessageTimestamp; } /** For recreating per-block ATStateData from repository where not all info is needed */ public ATStateData(String ATAddress, int height, byte[] stateHash, Long fees, boolean isInitial) { - this(ATAddress, height, null, null, stateHash, fees, isInitial); - } - - /** For creating ATStateData from serialized bytes when we don't have all the info */ - public ATStateData(String ATAddress, byte[] stateHash) { - // This won't ever be initial AT state from deployment as that's never serialized over the network, - // but generated when the DeployAtTransaction is processed locally. - this(ATAddress, null, null, null, stateHash, null, false); + this(ATAddress, height, null, stateHash, fees, isInitial, null); } /** For creating ATStateData from serialized bytes when we don't have all the info */ public ATStateData(String ATAddress, byte[] stateHash, Long fees) { - // This won't ever be initial AT state from deployment as that's never serialized over the network, - // but generated when the DeployAtTransaction is processed locally. - this(ATAddress, null, null, null, stateHash, fees, false); + // This won't ever be initial AT state from deployment, as that's never serialized over the network. + this(ATAddress, null, null, stateHash, fees, false, null); } // Getters / setters @@ -58,10 +53,6 @@ public void setHeight(Integer height) { this.height = height; } - public Long getCreation() { - return this.creation; - } - public byte[] getStateData() { return this.stateData; } @@ -78,4 +69,12 @@ public boolean isInitial() { return this.isInitial; } + public Long getSleepUntilMessageTimestamp() { + return this.sleepUntilMessageTimestamp; + } + + public void setSleepUntilMessageTimestamp(Long sleepUntilMessageTimestamp) { + this.sleepUntilMessageTimestamp = sleepUntilMessageTimestamp; + } + } diff --git a/src/main/java/org/qortal/data/block/BlockArchiveData.java b/src/main/java/org/qortal/data/block/BlockArchiveData.java new file mode 100644 index 000000000..c9db4032b --- /dev/null +++ b/src/main/java/org/qortal/data/block/BlockArchiveData.java @@ -0,0 +1,47 @@ +package org.qortal.data.block; + +import org.qortal.block.Block; + +public class BlockArchiveData { + + // Properties + private byte[] signature; + private Integer height; + private Long timestamp; + private byte[] minterPublicKey; + + // Constructors + + public BlockArchiveData(byte[] signature, Integer height, long timestamp, byte[] minterPublicKey) { + this.signature = signature; + this.height = height; + this.timestamp = timestamp; + this.minterPublicKey = minterPublicKey; + } + + public BlockArchiveData(BlockData blockData) { + this.signature = blockData.getSignature(); + this.height = blockData.getHeight(); + this.timestamp = blockData.getTimestamp(); + this.minterPublicKey = blockData.getMinterPublicKey(); + } + + // Getters/setters + + public byte[] getSignature() { + return this.signature; + } + + public Integer getHeight() { + return this.height; + } + + public Long getTimestamp() { + return this.timestamp; + } + + public byte[] getMinterPublicKey() { + return this.minterPublicKey; + } + +} diff --git a/src/main/java/org/qortal/data/block/BlockData.java b/src/main/java/org/qortal/data/block/BlockData.java index 63f40a475..61d1a7fb1 100644 --- a/src/main/java/org/qortal/data/block/BlockData.java +++ b/src/main/java/org/qortal/data/block/BlockData.java @@ -9,7 +9,10 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import org.qortal.block.BlockChain; +import org.qortal.settings.Settings; import org.qortal.crypto.Crypto; +import org.qortal.utils.NTP; // All properties to be converted to JSON via JAX-RS @XmlAccessorType(XmlAccessType.FIELD) @@ -79,6 +82,25 @@ public BlockData(int version, byte[] reference, int transactionCount, long total null, 0, null, null); } + public BlockData(BlockData other) { + this.version = other.version; + this.reference = other.reference; + this.transactionCount = other.transactionCount; + this.totalFees = other.totalFees; + this.transactionsSignature = other.transactionsSignature; + this.height = other.height; + this.timestamp = other.timestamp; + this.minterPublicKey = other.minterPublicKey; + this.minterSignature = other.minterSignature; + this.atCount = other.atCount; + this.atFees = other.atFees; + this.encodedOnlineAccounts = other.encodedOnlineAccounts; + this.onlineAccountsCount = other.onlineAccountsCount; + this.onlineAccountsTimestamp = other.onlineAccountsTimestamp; + this.onlineAccountsSignatures = other.onlineAccountsSignatures; + this.signature = other.signature; + } + // Getters/setters public byte[] getSignature() { @@ -185,6 +207,17 @@ public byte[] getOnlineAccountsSignatures() { return this.onlineAccountsSignatures; } + public void setOnlineAccountsSignatures(byte[] onlineAccountsSignatures) { + this.onlineAccountsSignatures = onlineAccountsSignatures; + } + + public boolean isTrimmed() { + long onlineAccountSignaturesTrimmedTimestamp = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime(); + long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime(); + long blockTimestamp = this.getTimestamp(); + return blockTimestamp < onlineAccountSignaturesTrimmedTimestamp && blockTimestamp < currentTrimmableTimestamp; + } + // JAXB special @XmlElement(name = "minterAddress") diff --git a/src/main/java/org/qortal/data/block/BlockSummaryData.java b/src/main/java/org/qortal/data/block/BlockSummaryData.java index 3d789dd61..2167f0f0a 100644 --- a/src/main/java/org/qortal/data/block/BlockSummaryData.java +++ b/src/main/java/org/qortal/data/block/BlockSummaryData.java @@ -2,8 +2,7 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; - -import org.qortal.transform.block.BlockTransformer; +import java.util.Arrays; @XmlAccessorType(XmlAccessType.FIELD) public class BlockSummaryData { @@ -14,6 +13,10 @@ public class BlockSummaryData { private byte[] minterPublicKey; private int onlineAccountsCount; + // Optional, set during construction + private Long timestamp; + private Integer transactionCount; + // Optional, set after construction private Integer minterLevel; @@ -29,17 +32,23 @@ public BlockSummaryData(int height, byte[] signature, byte[] minterPublicKey, in this.onlineAccountsCount = onlineAccountsCount; } + public BlockSummaryData(int height, byte[] signature, byte[] minterPublicKey, int onlineAccountsCount, long timestamp, int transactionCount) { + this.height = height; + this.signature = signature; + this.minterPublicKey = minterPublicKey; + this.onlineAccountsCount = onlineAccountsCount; + this.timestamp = timestamp; + this.transactionCount = transactionCount; + } + public BlockSummaryData(BlockData blockData) { this.height = blockData.getHeight(); this.signature = blockData.getSignature(); this.minterPublicKey = blockData.getMinterPublicKey(); + this.onlineAccountsCount = blockData.getOnlineAccountsCount(); - byte[] encodedOnlineAccounts = blockData.getEncodedOnlineAccounts(); - if (encodedOnlineAccounts != null) { - this.onlineAccountsCount = BlockTransformer.decodeOnlineAccounts(encodedOnlineAccounts).size(); - } else { - this.onlineAccountsCount = 0; - } + this.timestamp = blockData.getTimestamp(); + this.transactionCount = blockData.getTransactionCount(); } // Getters / setters @@ -60,6 +69,14 @@ public int getOnlineAccountsCount() { return this.onlineAccountsCount; } + public Long getTimestamp() { + return this.timestamp; + } + + public Integer getTransactionCount() { + return this.transactionCount; + } + public Integer getMinterLevel() { return this.minterLevel; } @@ -68,4 +85,21 @@ public void setMinterLevel(Integer minterLevel) { this.minterLevel = minterLevel; } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + BlockSummaryData otherBlockSummary = (BlockSummaryData) o; + if (this.getSignature() == null || otherBlockSummary.getSignature() == null) + return false; + + // Treat two block summaries as equal if they have matching signatures + return Arrays.equals(this.getSignature(), otherBlockSummary.getSignature()); + } + } diff --git a/src/main/java/org/qortal/data/block/CommonBlockData.java b/src/main/java/org/qortal/data/block/CommonBlockData.java new file mode 100644 index 000000000..dd502df71 --- /dev/null +++ b/src/main/java/org/qortal/data/block/CommonBlockData.java @@ -0,0 +1,56 @@ +package org.qortal.data.block; + +import org.qortal.data.network.PeerChainTipData; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.math.BigInteger; +import java.util.List; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CommonBlockData { + + // Properties + private BlockSummaryData commonBlockSummary = null; + private List blockSummariesAfterCommonBlock = null; + private BigInteger chainWeight = null; + private PeerChainTipData chainTipData = null; + + // Constructors + + protected CommonBlockData() { + } + + public CommonBlockData(BlockSummaryData commonBlockSummary, PeerChainTipData chainTipData) { + this.commonBlockSummary = commonBlockSummary; + this.chainTipData = chainTipData; + } + + + // Getters / setters + + public BlockSummaryData getCommonBlockSummary() { + return this.commonBlockSummary; + } + + public List getBlockSummariesAfterCommonBlock() { + return this.blockSummariesAfterCommonBlock; + } + + public void setBlockSummariesAfterCommonBlock(List blockSummariesAfterCommonBlock) { + this.blockSummariesAfterCommonBlock = blockSummariesAfterCommonBlock; + } + + public BigInteger getChainWeight() { + return this.chainWeight; + } + + public void setChainWeight(BigInteger chainWeight) { + this.chainWeight = chainWeight; + } + + public PeerChainTipData getChainTipData() { + return this.chainTipData; + } + +} diff --git a/src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java b/src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java index 8c9b66023..69250e544 100644 --- a/src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java +++ b/src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java @@ -4,14 +4,14 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import org.qortal.crosschain.AcctMode; + import io.swagger.v3.oas.annotations.media.Schema; // All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) public class CrossChainTradeData { - public enum Mode { OFFER, TRADE }; - // Properties @Schema(description = "AT's Qortal address") @@ -20,44 +20,79 @@ public enum Mode { OFFER, TRADE }; @Schema(description = "AT creator's Qortal address") public String qortalCreator; + @Schema(description = "AT creator's ephemeral trading key-pair represented as Qortal address") + public String qortalCreatorTradeAddress; + + @Deprecated + @Schema(description = "DEPRECATED: use creatorForeignPKH instead") + public byte[] creatorBitcoinPKH; + + @Schema(description = "AT creator's foreign blockchain trade public-key-hash (PKH)") + public byte[] creatorForeignPKH; + @Schema(description = "Timestamp when AT was created (milliseconds since epoch)") public long creationTimestamp; + @Schema(description = "Suggested trade timeout (minutes)", example = "10080") + public int tradeTimeout; + @Schema(description = "AT's current QORT balance") @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) public long qortBalance; - @Schema(description = "HASH160 of 32-byte secret") - public byte[] secretHash; + @Schema(description = "HASH160 of 32-byte secret-A") + public byte[] hashOfSecretA; - @Schema(description = "Initial QORT payment that will be sent to Qortal trade partner") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long initialPayout; + @Schema(description = "HASH160 of 32-byte secret-B") + public byte[] hashOfSecretB; @Schema(description = "Final QORT payment that will be sent to Qortal trade partner") @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long redeemPayout; + public long qortAmount; @Schema(description = "Trade partner's Qortal address (trade begins when this is set)") - public String qortalRecipient; + public String qortalPartnerAddress; @Schema(description = "Timestamp when AT switched to trade mode") public Long tradeModeTimestamp; - @Schema(description = "How long from beginning trade until AT triggers automatic refund to AT creator (minutes)") - public long tradeRefundTimeout; + @Schema(description = "How long from AT creation until AT triggers automatic refund to AT creator (minutes)") + public Integer refundTimeout; @Schema(description = "Actual Qortal block height when AT will automatically refund to AT creator (after trade begins)") public Integer tradeRefundHeight; - @Schema(description = "Amount, in BTC, that AT creator expects Bitcoin P2SH to pay out (excluding miner fees)") + @Deprecated + @Schema(description = "DEPRECATED: use expectedForeignAmount instread") @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) public long expectedBitcoin; - public Mode mode; + @Schema(description = "Amount, in foreign blockchain currency, that AT creator expects trade partner to pay out (excluding miner fees)") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long expectedForeignAmount; + + @Schema(description = "Current AT execution mode") + public AcctMode mode; - @Schema(description = "Suggested Bitcoin P2SH nLockTime based on trade timeout") - public Integer lockTime; + @Schema(description = "Suggested P2SH-A nLockTime based on trade timeout") + public Integer lockTimeA; + + @Schema(description = "Suggested P2SH-B nLockTime based on trade timeout") + public Integer lockTimeB; + + @Deprecated + @Schema(description = "DEPRECATED: use partnerForeignPKH instead") + public byte[] partnerBitcoinPKH; + + @Schema(description = "Trade partner's foreign blockchain public-key-hash (PKH)") + public byte[] partnerForeignPKH; + + @Schema(description = "Trade partner's Qortal receiving address") + public String qortalPartnerReceivingAddress; + + public String foreignBlockchain; + + public String acctName; // Constructors @@ -65,4 +100,10 @@ public enum Mode { OFFER, TRADE }; public CrossChainTradeData() { } + public void duplicateDeprecated() { + this.creatorBitcoinPKH = this.creatorForeignPKH; + this.expectedBitcoin = this.expectedForeignAmount; + this.partnerBitcoinPKH = this.partnerForeignPKH; + } + } diff --git a/src/main/java/org/qortal/data/crosschain/TradeBotData.java b/src/main/java/org/qortal/data/crosschain/TradeBotData.java new file mode 100644 index 000000000..194814668 --- /dev/null +++ b/src/main/java/org/qortal/data/crosschain/TradeBotData.java @@ -0,0 +1,268 @@ +package org.qortal.data.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import io.swagger.v3.oas.annotations.media.Schema; +import org.json.JSONObject; + +import org.qortal.utils.Base58; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class TradeBotData { + + private byte[] tradePrivateKey; + + private String acctName; + private String tradeState; + + // Internal use - not shown via API + @XmlTransient + @Schema(hidden = true) + private int tradeStateValue; + + private String creatorAddress; + private String atAddress; + + private long timestamp; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long qortAmount; + + private byte[] tradeNativePublicKey; + private byte[] tradeNativePublicKeyHash; + String tradeNativeAddress; + + private byte[] secret; + private byte[] hashOfSecret; + + private String foreignBlockchain; + private byte[] tradeForeignPublicKey; + private byte[] tradeForeignPublicKeyHash; + + @Deprecated + @Schema(description = "DEPRECATED: use foreignAmount instead", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long bitcoinAmount; + + @Schema(description = "amount in foreign blockchain currency", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long foreignAmount; + + // Never expose this via API + @XmlTransient + @Schema(hidden = true) + private String foreignKey; + + private byte[] lastTransactionSignature; + private Integer lockTimeA; + + // Could be Bitcoin or Qortal... + private byte[] receivingAccountInfo; + + protected TradeBotData() { + /* JAXB */ + } + + public TradeBotData(byte[] tradePrivateKey, String acctName, String tradeState, int tradeStateValue, + String creatorAddress, String atAddress, + long timestamp, long qortAmount, + byte[] tradeNativePublicKey, byte[] tradeNativePublicKeyHash, String tradeNativeAddress, + byte[] secret, byte[] hashOfSecret, + String foreignBlockchain, byte[] tradeForeignPublicKey, byte[] tradeForeignPublicKeyHash, + long foreignAmount, String foreignKey, + byte[] lastTransactionSignature, Integer lockTimeA, byte[] receivingAccountInfo) { + this.tradePrivateKey = tradePrivateKey; + this.acctName = acctName; + this.tradeState = tradeState; + this.tradeStateValue = tradeStateValue; + this.creatorAddress = creatorAddress; + this.atAddress = atAddress; + this.timestamp = timestamp; + this.qortAmount = qortAmount; + this.tradeNativePublicKey = tradeNativePublicKey; + this.tradeNativePublicKeyHash = tradeNativePublicKeyHash; + this.tradeNativeAddress = tradeNativeAddress; + this.secret = secret; + this.hashOfSecret = hashOfSecret; + this.foreignBlockchain = foreignBlockchain; + this.tradeForeignPublicKey = tradeForeignPublicKey; + this.tradeForeignPublicKeyHash = tradeForeignPublicKeyHash; + // deprecated copy + this.bitcoinAmount = foreignAmount; + this.foreignAmount = foreignAmount; + this.foreignKey = foreignKey; + this.lastTransactionSignature = lastTransactionSignature; + this.lockTimeA = lockTimeA; + this.receivingAccountInfo = receivingAccountInfo; + } + + public byte[] getTradePrivateKey() { + return this.tradePrivateKey; + } + + public String getAcctName() { + return this.acctName; + } + + public String getState() { + return this.tradeState; + } + + public void setState(String state) { + this.tradeState = state; + } + + public int getStateValue() { + return this.tradeStateValue; + } + + public void setStateValue(int stateValue) { + this.tradeStateValue = stateValue; + } + + public String getCreatorAddress() { + return this.creatorAddress; + } + + public String getAtAddress() { + return this.atAddress; + } + + public void setAtAddress(String atAddress) { + this.atAddress = atAddress; + } + + public long getTimestamp() { + return this.timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public long getQortAmount() { + return this.qortAmount; + } + + public byte[] getTradeNativePublicKey() { + return this.tradeNativePublicKey; + } + + public byte[] getTradeNativePublicKeyHash() { + return this.tradeNativePublicKeyHash; + } + + public String getTradeNativeAddress() { + return this.tradeNativeAddress; + } + + public byte[] getSecret() { + return this.secret; + } + + public byte[] getHashOfSecret() { + return this.hashOfSecret; + } + + public String getForeignBlockchain() { + return this.foreignBlockchain; + } + + public byte[] getTradeForeignPublicKey() { + return this.tradeForeignPublicKey; + } + + public byte[] getTradeForeignPublicKeyHash() { + return this.tradeForeignPublicKeyHash; + } + + public long getForeignAmount() { + return this.foreignAmount; + } + + public String getForeignKey() { + return this.foreignKey; + } + + public byte[] getLastTransactionSignature() { + return this.lastTransactionSignature; + } + + public void setLastTransactionSignature(byte[] lastTransactionSignature) { + this.lastTransactionSignature = lastTransactionSignature; + } + + public Integer getLockTimeA() { + return this.lockTimeA; + } + + public void setLockTimeA(Integer lockTimeA) { + this.lockTimeA = lockTimeA; + } + + public byte[] getReceivingAccountInfo() { + return this.receivingAccountInfo; + } + + public JSONObject toJson() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("tradePrivateKey", Base58.encode(this.getTradePrivateKey())); + jsonObject.put("acctName", this.getAcctName()); + jsonObject.put("tradeState", this.getState()); + jsonObject.put("tradeStateValue", this.getStateValue()); + jsonObject.put("creatorAddress", this.getCreatorAddress()); + jsonObject.put("atAddress", this.getAtAddress()); + jsonObject.put("timestamp", this.getTimestamp()); + jsonObject.put("qortAmount", this.getQortAmount()); + if (this.getTradeNativePublicKey() != null) jsonObject.put("tradeNativePublicKey", Base58.encode(this.getTradeNativePublicKey())); + if (this.getTradeNativePublicKeyHash() != null) jsonObject.put("tradeNativePublicKeyHash", Base58.encode(this.getTradeNativePublicKeyHash())); + jsonObject.put("tradeNativeAddress", this.getTradeNativeAddress()); + if (this.getSecret() != null) jsonObject.put("secret", Base58.encode(this.getSecret())); + if (this.getHashOfSecret() != null) jsonObject.put("hashOfSecret", Base58.encode(this.getHashOfSecret())); + jsonObject.put("foreignBlockchain", this.getForeignBlockchain()); + if (this.getTradeForeignPublicKey() != null) jsonObject.put("tradeForeignPublicKey", Base58.encode(this.getTradeForeignPublicKey())); + if (this.getTradeForeignPublicKeyHash() != null) jsonObject.put("tradeForeignPublicKeyHash", Base58.encode(this.getTradeForeignPublicKeyHash())); + jsonObject.put("foreignKey", this.getForeignKey()); + jsonObject.put("foreignAmount", this.getForeignAmount()); + if (this.getLastTransactionSignature() != null) jsonObject.put("lastTransactionSignature", Base58.encode(this.getLastTransactionSignature())); + jsonObject.put("lockTimeA", this.getLockTimeA()); + if (this.getReceivingAccountInfo() != null) jsonObject.put("receivingAccountInfo", Base58.encode(this.getReceivingAccountInfo())); + return jsonObject; + } + + public static TradeBotData fromJson(JSONObject json) { + return new TradeBotData( + json.isNull("tradePrivateKey") ? null : Base58.decode(json.getString("tradePrivateKey")), + json.isNull("acctName") ? null : json.getString("acctName"), + json.isNull("tradeState") ? null : json.getString("tradeState"), + json.isNull("tradeStateValue") ? null : json.getInt("tradeStateValue"), + json.isNull("creatorAddress") ? null : json.getString("creatorAddress"), + json.isNull("atAddress") ? null : json.getString("atAddress"), + json.isNull("timestamp") ? null : json.getLong("timestamp"), + json.isNull("qortAmount") ? null : json.getLong("qortAmount"), + json.isNull("tradeNativePublicKey") ? null : Base58.decode(json.getString("tradeNativePublicKey")), + json.isNull("tradeNativePublicKeyHash") ? null : Base58.decode(json.getString("tradeNativePublicKeyHash")), + json.isNull("tradeNativeAddress") ? null : json.getString("tradeNativeAddress"), + json.isNull("secret") ? null : Base58.decode(json.getString("secret")), + json.isNull("hashOfSecret") ? null : Base58.decode(json.getString("hashOfSecret")), + json.isNull("foreignBlockchain") ? null : json.getString("foreignBlockchain"), + json.isNull("tradeForeignPublicKey") ? null : Base58.decode(json.getString("tradeForeignPublicKey")), + json.isNull("tradeForeignPublicKeyHash") ? null : Base58.decode(json.getString("tradeForeignPublicKeyHash")), + json.isNull("foreignAmount") ? null : json.getLong("foreignAmount"), + json.isNull("foreignKey") ? null : json.getString("foreignKey"), + json.isNull("lastTransactionSignature") ? null : Base58.decode(json.getString("lastTransactionSignature")), + json.isNull("lockTimeA") ? null : json.getInt("lockTimeA"), + json.isNull("receivingAccountInfo") ? null : Base58.decode(json.getString("receivingAccountInfo")) + ); + } + + // Mostly for debugging + public String toString() { + return String.format("%s: %s (%d)", this.atAddress, this.tradeState, this.tradeStateValue); + } + +} diff --git a/src/main/java/org/qortal/data/network/ArbitraryPeerData.java b/src/main/java/org/qortal/data/network/ArbitraryPeerData.java new file mode 100644 index 000000000..30f8cf24e --- /dev/null +++ b/src/main/java/org/qortal/data/network/ArbitraryPeerData.java @@ -0,0 +1,112 @@ +package org.qortal.data.network; + +import com.google.common.net.InetAddresses; +import org.qortal.crypto.Crypto; +import org.qortal.network.Peer; +import org.qortal.utils.NTP; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +public class ArbitraryPeerData { + + private final byte[] hash; + private final String peerAddress; + private Integer successes; + private Integer failures; + private Long lastAttempted; + private Long lastRetrieved; + + public ArbitraryPeerData(byte[] hash, String peerAddress, Integer successes, + Integer failures, Long lastAttempted, Long lastRetrieved) { + this.hash = hash; + this.peerAddress = peerAddress; + this.successes = successes; + this.failures = failures; + this.lastAttempted = lastAttempted; + this.lastRetrieved = lastRetrieved; + } + + public ArbitraryPeerData(byte[] signature, Peer peer) { + this(Crypto.digest(signature), peer.getPeerData().getAddress().toString(), + 0, 0, 0L, 0L); + } + + public ArbitraryPeerData(byte[] signature, String peerAddress) { + this(Crypto.digest(signature), peerAddress, 0, 0, 0L, 0L); + } + + public boolean isPeerAddressValid() { + // Validate the peer address to prevent arbitrary values being added to the db + String[] parts = this.peerAddress.split(":"); + if (parts.length != 2) { + // Invalid format + return false; + } + String host = parts[0]; + if (!InetAddresses.isInetAddress(host)) { + // Invalid host + return false; + } + int port = Integer.valueOf(parts[1]); + if (port <= 0 || port > 65535) { + // Invalid port + return false; + } + + // Make sure that it's not a local address + try { + InetAddress addr = InetAddress.getByName(host); + if (addr.isLoopbackAddress() || addr.isLinkLocalAddress() || addr.isSiteLocalAddress()) { + // Ignore local addresses + return false; + } + } catch (UnknownHostException e) { + return false; + } + + // Valid host/port combination + return true; + } + + public void incrementSuccesses() { + this.successes++; + } + + public void incrementFailures() { + this.failures++; + } + + public void markAsAttempted() { + this.lastAttempted = NTP.getTime(); + } + + public void markAsRetrieved() { + this.lastRetrieved = NTP.getTime(); + } + + public byte[] getHash() { + return this.hash; + } + + public String getPeerAddress() { + return this.peerAddress; + } + + public Integer getSuccesses() { + return this.successes; + } + + public Integer getFailures() { + return this.failures; + } + + public Long getLastAttempted() { + return this.lastAttempted; + } + + public Long getLastRetrieved() { + return this.lastRetrieved; + } + +} diff --git a/src/main/java/org/qortal/data/network/OnlineAccountLevel.java b/src/main/java/org/qortal/data/network/OnlineAccountLevel.java new file mode 100644 index 000000000..0589b2390 --- /dev/null +++ b/src/main/java/org/qortal/data/network/OnlineAccountLevel.java @@ -0,0 +1,55 @@ +package org.qortal.data.network; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class OnlineAccountLevel { + + protected int level; + protected int count; + + // Constructors + + // necessary for JAXB serialization + protected OnlineAccountLevel() { + } + + public OnlineAccountLevel(int level, int count) { + this.level = level; + this.count = count; + } + + public int getLevel() { + return this.level; + } + + public int getCount() { + return this.count; + } + + public void setCount(int count) { + this.count = count; + } + + + // Comparison + + @Override + public boolean equals(Object other) { + if (other == this) + return true; + + if (!(other instanceof OnlineAccountLevel)) + return false; + + OnlineAccountLevel otherOnlineAccountData = (OnlineAccountLevel) other; + + if (otherOnlineAccountData.level != this.level) + return false; + + return true; + } +} diff --git a/src/main/java/org/qortal/data/network/PeerData.java b/src/main/java/org/qortal/data/network/PeerData.java index 3362ff113..09982c000 100644 --- a/src/main/java/org/qortal/data/network/PeerData.java +++ b/src/main/java/org/qortal/data/network/PeerData.java @@ -13,6 +13,8 @@ @XmlAccessorType(XmlAccessType.FIELD) public class PeerData { + public static final int MAX_PEER_ADDRESS_SIZE = 255; + // Properties // Don't expose this via JAXB - use pretty getter instead diff --git a/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java b/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java index 565298521..acd5c3a6b 100644 --- a/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java +++ b/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java @@ -1,17 +1,22 @@ package org.qortal.data.transaction; import java.util.List; +import java.util.Map; import javax.xml.bind.Unmarshaller; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import org.eclipse.persistence.oxm.annotations.XmlDiscriminatorValue; +import org.qortal.arbitrary.misc.Service; import org.qortal.data.PaymentData; import org.qortal.transaction.Transaction.TransactionType; import io.swagger.v3.oas.annotations.media.Schema; +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + // All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) @Schema(allOf = { TransactionData.class }) @@ -25,17 +30,65 @@ public enum DataType { DATA_HASH; } + // Methods + public enum Method { + PUT(0), // A complete replacement of a resource + PATCH(1); // An update / partial replacement of a resource + + public final int value; + + private static final Map map = stream(Method.values()) + .collect(toMap(method -> method.value, method -> method)); + + Method(int value) { + this.value = value; + } + + public static Method valueOf(int value) { + return map.get(value); + } + } + + // Compression types + public enum Compression { + NONE(0), + ZIP(1); + + public final int value; + + private static final Map map = stream(Compression.values()) + .collect(toMap(compression -> compression.value, compression -> compression)); + + Compression(int value) { + this.value = value; + } + + public static Compression valueOf(int value) { + return map.get(value); + } + } + // Properties private int version; - @Schema(example = "sender_public_key") private byte[] senderPublicKey; - private int service; + private Service service; + private int nonce; + private int size; + + private String name; + private String identifier; + private Method method; + private byte[] secret; + private Compression compression; @Schema(example = "raw_data_in_base58") private byte[] data; private DataType dataType; + @Schema(example = "metadata_file_hash_in_base58") + private byte[] metadataHash; + private List payments; // Constructors @@ -50,14 +103,24 @@ public void afterUnmarshal(Unmarshaller u, Object parent) { } public ArbitraryTransactionData(BaseTransactionData baseTransactionData, - int version, int service, byte[] data, DataType dataType, List payments) { + int version, Service service, int nonce, int size, + String name, String identifier, Method method, byte[] secret, Compression compression, + byte[] data, DataType dataType, byte[] metadataHash, List payments) { super(TransactionType.ARBITRARY, baseTransactionData); this.senderPublicKey = baseTransactionData.creatorPublicKey; this.version = version; this.service = service; + this.nonce = nonce; + this.size = size; + this.name = name; + this.identifier = identifier; + this.method = method; + this.secret = secret; + this.compression = compression; this.data = data; this.dataType = dataType; + this.metadataHash = metadataHash; this.payments = payments; } @@ -71,10 +134,42 @@ public int getVersion() { return this.version; } - public int getService() { + public Service getService() { return this.service; } + public int getNonce() { + return this.nonce; + } + + public void setNonce(int nonce) { + this.nonce = nonce; + } + + public int getSize() { + return this.size; + } + + public String getName() { + return this.name; + } + + public String getIdentifier() { + return (this.identifier != "") ? this.identifier : null; + } + + public Method getMethod() { + return this.method; + } + + public byte[] getSecret() { + return this.secret; + } + + public Compression getCompression() { + return this.compression; + } + public byte[] getData() { return this.data; } @@ -91,6 +186,14 @@ public void setDataType(DataType dataType) { this.dataType = dataType; } + public byte[] getMetadataHash() { + return this.metadataHash; + } + + public void setMetadataHash(byte[] metadataHash) { + this.metadataHash = metadataHash; + } + public List getPayments() { return this.payments; } diff --git a/src/main/java/org/qortal/data/transaction/PresenceTransactionData.java b/src/main/java/org/qortal/data/transaction/PresenceTransactionData.java new file mode 100644 index 000000000..001bd5b4b --- /dev/null +++ b/src/main/java/org/qortal/data/transaction/PresenceTransactionData.java @@ -0,0 +1,73 @@ +package org.qortal.data.transaction; + +import javax.xml.bind.Unmarshaller; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import org.qortal.transaction.PresenceTransaction.PresenceType; +import org.qortal.transaction.Transaction.TransactionType; + +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.media.Schema.AccessMode; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +@Schema(allOf = { TransactionData.class }) +public class PresenceTransactionData extends TransactionData { + + // Properties + @Schema(description = "sender's public key", example = "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP") + private byte[] senderPublicKey; + + @Schema(accessMode = AccessMode.READ_ONLY) + private int nonce; + + private PresenceType presenceType; + + @Schema(description = "timestamp signature", example = "2yGEbwRFyhPZZckKA") + private byte[] timestampSignature; + + // Constructors + + // For JAXB + protected PresenceTransactionData() { + super(TransactionType.PRESENCE); + } + + public void afterUnmarshal(Unmarshaller u, Object parent) { + this.creatorPublicKey = this.senderPublicKey; + } + + public PresenceTransactionData(BaseTransactionData baseTransactionData, + int nonce, PresenceType presenceType, byte[] timestampSignature) { + super(TransactionType.PRESENCE, baseTransactionData); + + this.senderPublicKey = baseTransactionData.creatorPublicKey; + this.nonce = nonce; + this.presenceType = presenceType; + this.timestampSignature = timestampSignature; + } + + // Getters/Setters + + public byte[] getSenderPublicKey() { + return this.senderPublicKey; + } + + public int getNonce() { + return this.nonce; + } + + public void setNonce(int nonce) { + this.nonce = nonce; + } + + public PresenceType getPresenceType() { + return this.presenceType; + } + + public byte[] getTimestampSignature() { + return this.timestampSignature; + } + +} diff --git a/src/main/java/org/qortal/data/transaction/RegisterNameTransactionData.java b/src/main/java/org/qortal/data/transaction/RegisterNameTransactionData.java index d4455da19..c2b06fd22 100644 --- a/src/main/java/org/qortal/data/transaction/RegisterNameTransactionData.java +++ b/src/main/java/org/qortal/data/transaction/RegisterNameTransactionData.java @@ -26,7 +26,7 @@ public class RegisterNameTransactionData extends TransactionData { @Schema(description = "requested name", example = "my-name") private String name; - @Schema(description = "simple name-related info in JSON format", example = "{ \"age\": 30 }") + @Schema(description = "simple name-related info in JSON or text format", example = "Registered Name on the Qortal Chain") private String data; // For internal use diff --git a/src/main/java/org/qortal/data/transaction/TransactionData.java b/src/main/java/org/qortal/data/transaction/TransactionData.java index 397693b85..060901f2f 100644 --- a/src/main/java/org/qortal/data/transaction/TransactionData.java +++ b/src/main/java/org/qortal/data/transaction/TransactionData.java @@ -40,7 +40,7 @@ GroupApprovalTransactionData.class, SetGroupTransactionData.class, UpdateAssetTransactionData.class, AccountFlagsTransactionData.class, RewardShareTransactionData.class, - AccountLevelTransactionData.class, ChatTransactionData.class + AccountLevelTransactionData.class, ChatTransactionData.class, PresenceTransactionData.class }) //All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) diff --git a/src/main/java/org/qortal/data/transaction/UpdateNameTransactionData.java b/src/main/java/org/qortal/data/transaction/UpdateNameTransactionData.java index 43c8da593..b43361db4 100644 --- a/src/main/java/org/qortal/data/transaction/UpdateNameTransactionData.java +++ b/src/main/java/org/qortal/data/transaction/UpdateNameTransactionData.java @@ -26,7 +26,7 @@ public class UpdateNameTransactionData extends TransactionData { @Schema(description = "new name", example = "my-new-name") private String newName; - @Schema(description = "replacement simple name-related info in JSON format", example = "{ \"age\": 30 }") + @Schema(description = "replacement simple name-related info in JSON or text format", example = "Registered Name on the Qortal Chain") private String newData; // For internal use diff --git a/src/main/java/org/qortal/event/Event.java b/src/main/java/org/qortal/event/Event.java new file mode 100644 index 000000000..0c97522cb --- /dev/null +++ b/src/main/java/org/qortal/event/Event.java @@ -0,0 +1,5 @@ +package org.qortal.event; + +public interface Event { + +} diff --git a/src/main/java/org/qortal/event/EventBus.java b/src/main/java/org/qortal/event/EventBus.java new file mode 100644 index 000000000..6114c2c6d --- /dev/null +++ b/src/main/java/org/qortal/event/EventBus.java @@ -0,0 +1,65 @@ +package org.qortal.event; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public enum EventBus { + INSTANCE; + + private static final Logger LOGGER = LogManager.getLogger(EventBus.class); + + private static final List LISTENERS = new ArrayList<>(); + + public void addListener(Listener newListener) { + synchronized (LISTENERS) { + LISTENERS.add(newListener); + } + } + + public void removeListener(Listener listener) { + synchronized (LISTENERS) { + LISTENERS.remove(listener); + } + } + + /** + * WARNING: before calling this method, + * make sure current thread's repository session + * holds no locks, e.g. by calling + * repository.saveChanges() or + * repository.discardChanges(). + *

+ * This is because event listeners might open a new + * repository session which will deadlock HSQLDB + * if it tries to CHECKPOINT. + *

+ * The HSQLDB deadlock path is: + *

    + *
  • write-log blockchain.log has grown past CHECKPOINT threshold (50MB)
    • + *
  • alternatively, another thread has explicitly requested CHECKPOINT
    • + *
  • HSQLDB won't begin CHECKPOINT until all pending (SQL) transactions are committed or rolled back
    • + *
  • Same thread calls EventBus.INSTANCE.notify() before (SQL) transaction closed
    • + *
  • EventBus listener (same thread) requests a new repository session via RepositoryManager.getRepository()
    • + *
  • New repository sessions are blocked pending completion of CHECKPOINT
    • + *
  • Caller is blocked so never has a chance to close (SQL) transaction - hence deadlock
    • + *
+ */ + public void notify(Event event) { + List clonedListeners; + + synchronized (LISTENERS) { + clonedListeners = new ArrayList<>(LISTENERS); + } + + for (Listener listener : clonedListeners) + try { + listener.listen(event); + } catch (Exception e) { + // We don't want one listener to break other listeners, or caller + LOGGER.warn(() -> String.format("Caught %s from a listener processing %s", e.getClass().getSimpleName(), event.getClass().getSimpleName()), e); + } + } +} diff --git a/src/main/java/org/qortal/event/Listener.java b/src/main/java/org/qortal/event/Listener.java new file mode 100644 index 000000000..cb1668bf2 --- /dev/null +++ b/src/main/java/org/qortal/event/Listener.java @@ -0,0 +1,6 @@ +package org.qortal.event; + +@FunctionalInterface +public interface Listener { + void listen(Event event); +} diff --git a/src/main/java/org/qortal/globalization/Translator.java b/src/main/java/org/qortal/globalization/Translator.java index 8f0b61366..6481dde71 100644 --- a/src/main/java/org/qortal/globalization/Translator.java +++ b/src/main/java/org/qortal/globalization/Translator.java @@ -10,12 +10,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.settings.Settings; public enum Translator { INSTANCE; private static final Logger LOGGER = LogManager.getLogger(Translator.class); - private static final String DEFAULT_LANG = Locale.getDefault().getLanguage(); private static final Map resourceBundles = new HashMap<>(); @@ -34,7 +34,7 @@ public String translate(String className, String lang, String key, Object... arg } public String translate(String className, String key) { - return this.translate(className, DEFAULT_LANG, key); + return this.translate(className, Settings.getInstance().getLocaleLang(), key); } public Set keySet(String className, String lang) { diff --git a/src/main/java/org/qortal/gui/Gui.java b/src/main/java/org/qortal/gui/Gui.java index 118718e2e..87342f6a9 100644 --- a/src/main/java/org/qortal/gui/Gui.java +++ b/src/main/java/org/qortal/gui/Gui.java @@ -23,17 +23,21 @@ public class Gui { private SysTray sysTray = null; private Gui() { - this.isHeadless = GraphicsEnvironment.isHeadless(); - - if (!this.isHeadless) { - try { - UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()); - } catch (ClassNotFoundException | InstantiationException | IllegalAccessException - | UnsupportedLookAndFeelException e) { - // Use whatever look-and-feel comes by default then + try { + this.isHeadless = GraphicsEnvironment.isHeadless(); + + if (!this.isHeadless) { + try { + UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()); + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException + | UnsupportedLookAndFeelException e) { + // Use whatever look-and-feel comes by default then + } + + showSplash(); } - - showSplash(); + } catch (Exception e) { + LOGGER.info("Unable to initialize GUI: {}", e.getMessage()); } } diff --git a/src/main/java/org/qortal/gui/SplashFrame.java b/src/main/java/org/qortal/gui/SplashFrame.java index e08590309..c4ea51d09 100644 --- a/src/main/java/org/qortal/gui/SplashFrame.java +++ b/src/main/java/org/qortal/gui/SplashFrame.java @@ -1,64 +1,98 @@ package org.qortal.gui; -import java.awt.BorderLayout; -import java.awt.Image; +import java.awt.*; import java.util.ArrayList; import java.util.List; -import java.awt.Dimension; -import java.awt.Graphics; import java.awt.image.BufferedImage; -import javax.swing.JDialog; -import javax.swing.JPanel; +import javax.swing.*; +import javax.swing.border.EmptyBorder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; public class SplashFrame { protected static final Logger LOGGER = LogManager.getLogger(SplashFrame.class); private static SplashFrame instance; - private JDialog splashDialog; + private JFrame splashDialog; + private SplashPanel splashPanel; @SuppressWarnings("serial") public static class SplashPanel extends JPanel { private BufferedImage image; + private String defaultSplash = "Qlogo_512.png"; + + private JLabel statusLabel; + public SplashPanel() { - image = Gui.loadImage("splash.png"); - this.setPreferredSize(new Dimension(image.getWidth(), image.getHeight())); - this.setLayout(new BorderLayout()); + image = Gui.loadImage(defaultSplash); + + setOpaque(true); + setLayout(new BoxLayout(this, BoxLayout.Y_AXIS)); + setBorder(new EmptyBorder(10, 10, 10, 10)); + setBackground(Color.BLACK); + + // Add logo + JLabel imageLabel = new JLabel(new ImageIcon(image)); + imageLabel.setSize(new Dimension(300, 300)); + add(imageLabel); + + // Add spacing + add(Box.createRigidArea(new Dimension(0, 16))); + + // Add status label + String text = String.format("Starting Qortal Core v%s...", Controller.getInstance().getVersionStringWithoutPrefix()); + statusLabel = new JLabel(text, JLabel.CENTER); + statusLabel.setMaximumSize(new Dimension(500, 50)); + statusLabel.setFont(new Font("Verdana", Font.PLAIN, 20)); + statusLabel.setBackground(Color.BLACK); + statusLabel.setForeground(new Color(255, 255, 255, 255)); + statusLabel.setOpaque(true); + statusLabel.setBorder(null); + add(statusLabel); } @Override - protected void paintComponent(Graphics g) { - super.paintComponent(g); - g.drawImage(image, 0, 0, null); + public Dimension getPreferredSize() { + return new Dimension(500, 580); + } + + public void updateStatus(String text) { + if (statusLabel != null) { + statusLabel.setText(text); + } } } private SplashFrame() { - this.splashDialog = new JDialog(); + if (GraphicsEnvironment.isHeadless()) { + return; + } + + this.splashDialog = new JFrame(); List icons = new ArrayList<>(); icons.add(Gui.loadImage("icons/icon16.png")); - icons.add(Gui.loadImage("icons/icon32.png")); + icons.add(Gui.loadImage("icons/qortal_ui_tray_synced.png")); + icons.add(Gui.loadImage("icons/qortal_ui_tray_syncing_time-alt.png")); + icons.add(Gui.loadImage("icons/qortal_ui_tray_minting.png")); + icons.add(Gui.loadImage("icons/qortal_ui_tray_syncing.png")); icons.add(Gui.loadImage("icons/icon64.png")); - icons.add(Gui.loadImage("icons/icon128.png")); + icons.add(Gui.loadImage("icons/Qlogo_128.png")); this.splashDialog.setIconImages(icons); - this.splashDialog.setDefaultCloseOperation(JDialog.DO_NOTHING_ON_CLOSE); - this.splashDialog.setTitle("qortal"); - this.splashDialog.setContentPane(new SplashPanel()); - + this.splashPanel = new SplashPanel(); + this.splashDialog.getContentPane().add(this.splashPanel); + this.splashDialog.setDefaultCloseOperation(JFrame.DO_NOTHING_ON_CLOSE); this.splashDialog.setUndecorated(true); - this.splashDialog.setModal(false); this.splashDialog.pack(); this.splashDialog.setLocationRelativeTo(null); - this.splashDialog.toFront(); + this.splashDialog.setBackground(Color.BLACK); this.splashDialog.setVisible(true); - this.splashDialog.repaint(); } public static SplashFrame getInstance() { @@ -76,4 +110,10 @@ public void dispose() { this.splashDialog.dispose(); } + public void updateStatus(String text) { + if (this.splashPanel != null) { + this.splashPanel.updateStatus(text); + } + } + } diff --git a/src/main/java/org/qortal/gui/SysTray.java b/src/main/java/org/qortal/gui/SysTray.java index c456d6fe7..7a24f825b 100644 --- a/src/main/java/org/qortal/gui/SysTray.java +++ b/src/main/java/org/qortal/gui/SysTray.java @@ -61,7 +61,7 @@ private SysTray() { this.popupMenu = createJPopupMenu(); // Build TrayIcon without AWT PopupMenu (which doesn't support Unicode)... - this.trayIcon = new TrayIcon(Gui.loadImage("icons/icon32.png"), "qortal", null); + this.trayIcon = new TrayIcon(Gui.loadImage("icons/qortal_ui_tray_synced.png"), "qortal", null); // ...and attach mouse listener instead so we can use JPopupMenu (which does support Unicode) this.trayIcon.addMouseListener(new MouseAdapter() { @Override @@ -147,13 +147,13 @@ public void popupMenuCanceled(PopupMenuEvent e) { } }); - JMenuItem openUi = new JMenuItem(Translator.INSTANCE.translate("SysTray", "OPEN_UI")); + /* JMenuItem openUi = new JMenuItem(Translator.INSTANCE.translate("SysTray", "OPEN_UI")); openUi.addActionListener(actionEvent -> { destroyHiddenDialog(); new OpenUiWorker().execute(); }); - menu.add(openUi); + menu.add(openUi); */ JMenuItem openTimeCheck = new JMenuItem(Translator.INSTANCE.translate("SysTray", "CHECK_TIME_ACCURACY")); openTimeCheck.addActionListener(actionEvent -> { @@ -289,6 +289,29 @@ public void setToolTipText(String text) { this.trayIcon.setToolTip(text); } + public void setTrayIcon(int iconid) { + try { + if (trayIcon != null) { + switch (iconid) { + case 1: + this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_syncing_time-alt.png")); + break; + case 2: + this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_minting.png")); + break; + case 3: + this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_syncing.png")); + break; + case 4: + this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_synced.png")); + break; + } + } + } catch (Exception e) { + LOGGER.info("Unable to set tray icon: {}", e.getMessage()); + } + } + public void dispose() { if (trayIcon != null) SystemTray.getSystemTray().remove(this.trayIcon); diff --git a/src/main/java/org/qortal/list/ResourceList.java b/src/main/java/org/qortal/list/ResourceList.java new file mode 100644 index 000000000..099aa168e --- /dev/null +++ b/src/main/java/org/qortal/list/ResourceList.java @@ -0,0 +1,168 @@ +package org.qortal.list; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.qortal.settings.Settings; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class ResourceList { + + private static final Logger LOGGER = LogManager.getLogger(ResourceList.class); + + private String name; + private List list = Collections.synchronizedList(new ArrayList<>()); + + /** + * ResourceList + * Creates or updates a list for the purpose of tracking resources on the Qortal network + * This can be used for local blocking, or even for curating and sharing content lists + * Lists are backed off to JSON files (in the lists folder) to ease sharing between nodes and users + * + * @param name - the name of the list, for instance "blockedAddresses" + * @throws IOException + */ + public ResourceList(String name) throws IOException { + this.name = name; + this.load(); + } + + + /* Filesystem */ + + private Path getFilePath() { + String pathString = String.format("%s.json", Paths.get(Settings.getInstance().getListsPath(), this.name)); + return Paths.get(pathString); + } + + public void save() throws IOException { + if (this.name == null) { + throw new IllegalStateException("Can't save list with missing name"); + } + String jsonString = ResourceList.listToJSONString(this.list); + Path filePath = this.getFilePath(); + + // Create parent directory if needed + try { + Files.createDirectories(filePath.getParent()); + } catch (IOException e) { + throw new IllegalStateException("Unable to create lists directory"); + } + + BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toString())); + writer.write(jsonString); + writer.close(); + } + + private boolean load() throws IOException { + Path path = this.getFilePath(); + File resourceListFile = new File(path.toString()); + if (!resourceListFile.exists()) { + return false; + } + + try { + String jsonString = new String(Files.readAllBytes(path)); + this.list = ResourceList.listFromJSONString(jsonString); + } catch (IOException e) { + throw new IOException(String.format("Couldn't read contents from file %s", path.toString())); + } + + return true; + } + + public boolean revert() { + try { + return this.load(); + } catch (IOException e) { + LOGGER.info("Unable to revert list {}: {}", this.name, e.getMessage()); + } + return false; + } + + + /* List management */ + + public void add(String resource) { + if (resource == null || this.list == null) { + return; + } + if (!this.contains(resource, true)) { + this.list.add(resource); + } + } + + public void remove(String resource) { + if (resource == null || this.list == null) { + return; + } + this.list.remove(resource); + } + + public boolean contains(String resource, boolean caseSensitive) { + if (resource == null || this.list == null) { + return false; + } + + if (caseSensitive) { + return this.list.contains(resource); + } + else { + return this.list.stream().anyMatch(resource::equalsIgnoreCase); + } + } + + + /* Utils */ + + public static String listToJSONString(List list) { + if (list == null) { + return null; + } + JSONArray items = new JSONArray(); + for (String item : list) { + items.put(item); + } + return items.toString(4); + } + + private static List listFromJSONString(String jsonString) { + if (jsonString == null) { + return null; + } + JSONArray jsonList = new JSONArray(jsonString); + List resourceList = new ArrayList<>(); + for (int i=0; i getList() { + return this.list; + } + + public String toString() { + return this.name; + } + +} diff --git a/src/main/java/org/qortal/list/ResourceListManager.java b/src/main/java/org/qortal/list/ResourceListManager.java new file mode 100644 index 000000000..4182f87c9 --- /dev/null +++ b/src/main/java/org/qortal/list/ResourceListManager.java @@ -0,0 +1,144 @@ +package org.qortal.list; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class ResourceListManager { + + private static final Logger LOGGER = LogManager.getLogger(ResourceListManager.class); + + private static ResourceListManager instance; + private List lists = Collections.synchronizedList(new ArrayList<>()); + + + public ResourceListManager() { + } + + public static synchronized ResourceListManager getInstance() { + if (instance == null) { + instance = new ResourceListManager(); + } + return instance; + } + + private ResourceList getList(String listName) { + for (ResourceList list : this.lists) { + if (Objects.equals(list.getName(), listName)) { + return list; + } + } + + // List doesn't exist in array yet, so create it + // This will load any existing data from the filesystem + try { + ResourceList list = new ResourceList(listName); + this.lists.add(list); + return list; + + } catch (IOException e) { + LOGGER.info("Unable to load or create list {}: {}", listName, e.getMessage()); + return null; + } + + } + + public boolean addToList(String listName, String item, boolean save) { + ResourceList list = this.getList(listName); + if (list == null) { + return false; + } + + try { + list.add(item); + if (save) { + list.save(); + } + return true; + + } catch (IllegalStateException | IOException e) { + LOGGER.info(String.format("Unable to add item %s to list %s", item, list), e); + return false; + } + } + + public boolean removeFromList(String listName, String item, boolean save) { + ResourceList list = this.getList(listName); + if (list == null) { + return false; + } + + try { + list.remove(item); + + if (save) { + list.save(); + } + return true; + + } catch (IllegalStateException | IOException e) { + LOGGER.info(String.format("Unable to remove item %s from list %s", item, list), e); + return false; + } + } + + public boolean listContains(String listName, String item, boolean caseSensitive) { + ResourceList list = this.getList(listName); + if (list == null) { + return false; + } + return list.contains(item, caseSensitive); + } + + public void saveList(String listName) { + ResourceList list = this.getList(listName); + if (list == null) { + return; + } + + try { + list.save(); + } catch (IOException e) { + LOGGER.info("Unable to save list {} - reverting back to last saved state", list); + list.revert(); + } + } + + public void revertList(String listName) { + ResourceList list = this.getList(listName); + if (list == null) { + return; + } + list.revert(); + } + + public String getJSONStringForList(String listName) { + ResourceList list = this.getList(listName); + if (list == null) { + return null; + } + return list.getJSONString(); + } + + public List getStringsInList(String listName) { + ResourceList list = this.getList(listName); + if (list == null) { + return null; + } + return list.getList(); + } + + public int getItemCountForList(String listName) { + ResourceList list = this.getList(listName); + if (list == null) { + return 0; + } + return list.getList().size(); + } + +} diff --git a/src/main/java/org/qortal/naming/Name.java b/src/main/java/org/qortal/naming/Name.java index c372a8e3e..b27e9454e 100644 --- a/src/main/java/org/qortal/naming/Name.java +++ b/src/main/java/org/qortal/naming/Name.java @@ -78,9 +78,10 @@ public void update(UpdateNameTransactionData updateNameTransactionData) throws D // Set name's last-updated timestamp this.nameData.setUpdated(updateNameTransactionData.getTimestamp()); - // Update name and data where appropriate + // Update name, reduced name, and data where appropriate if (!updateNameTransactionData.getNewName().isEmpty()) { this.nameData.setName(updateNameTransactionData.getNewName()); + this.nameData.setReducedName(updateNameTransactionData.getReducedNewName()); // If we're changing the name, we need to delete old entry this.repository.getNameRepository().delete(updateNameTransactionData.getName()); @@ -106,6 +107,9 @@ public void revert(UpdateNameTransactionData updateNameTransactionData) throws D // We can find previous 'name' from update transaction this.nameData.setName(updateNameTransactionData.getName()); + // We can derive the previous 'reduced name' from the previous name + this.nameData.setReducedName(Unicode.sanitize(updateNameTransactionData.getName())); + // We might need to hunt for previous data value if (!updateNameTransactionData.getNewData().isEmpty()) this.nameData.setData(findPreviousData(nameReference)); @@ -261,4 +265,8 @@ private Long fetchPreviousUpdateTimestamp(byte[] nameReference) throws DataExcep return previousTransactionData.getTimestamp(); } + public NameData getNameData() { + return this.nameData; + } + } diff --git a/src/main/java/org/qortal/network/Handshake.java b/src/main/java/org/qortal/network/Handshake.java index 7f18e11b7..cdcff1d7e 100644 --- a/src/main/java/org/qortal/network/Handshake.java +++ b/src/main/java/org/qortal/network/Handshake.java @@ -4,7 +4,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -49,9 +48,12 @@ public Handshake onMessage(Peer peer, Message message) { return null; } + // Make a note of the senderPeerAddress, as this should be our public IP + Network.getInstance().ourPeerAddressUpdated(helloMessage.getSenderPeerAddress()); + String versionString = helloMessage.getVersionString(); - Matcher matcher = VERSION_PATTERN.matcher(versionString); + Matcher matcher = peer.VERSION_PATTERN.matcher(versionString); if (!matcher.lookingAt()) { LOGGER.debug(() -> String.format("Peer %s sent invalid HELLO version string '%s'", peer, versionString)); return null; @@ -72,6 +74,21 @@ public Handshake onMessage(Peer peer, Message message) { peer.setPeersConnectionTimestamp(peersConnectionTimestamp); peer.setPeersVersion(versionString, version); + // Ensure the peer is running at least the version specified in MIN_PEER_VERSION + if (peer.isAtLeastVersion(MIN_PEER_VERSION) == false) { + LOGGER.debug(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString)); + return null; + } + + if (Settings.getInstance().getAllowConnectionsWithOlderPeerVersions() == false) { + // Ensure the peer is running at least the minimum version allowed for connections + final String minPeerVersion = Settings.getInstance().getMinPeerVersion(); + if (peer.isAtLeastVersion(minPeerVersion) == false) { + LOGGER.debug(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString)); + return null; + } + } + return CHALLENGE; } @@ -79,8 +96,9 @@ public Handshake onMessage(Peer peer, Message message) { public void action(Peer peer) { String versionString = Controller.getInstance().getVersionString(); long timestamp = NTP.getTime(); + String senderPeerAddress = peer.getPeerData().getAddress().toString(); - Message helloMessage = new HelloMessage(timestamp, versionString); + Message helloMessage = new HelloMessage(timestamp, versionString, senderPeerAddress); if (!peer.sendMessage(helloMessage)) peer.disconnect("failed to send HELLO"); } @@ -162,7 +180,9 @@ public Handshake onMessage(Peer peer, Message message) { } int nonce = responseMessage.getNonce(); - if (!MemoryPoW.verify2(data, POW_BUFFER_SIZE, POW_DIFFICULTY, nonce)) { + int powBufferSize = peer.getPeersVersion() < PEER_VERSION_131 ? POW_BUFFER_SIZE_PRE_131 : POW_BUFFER_SIZE_POST_131; + int powDifficulty = peer.getPeersVersion() < PEER_VERSION_131 ? POW_DIFFICULTY_PRE_131 : POW_DIFFICULTY_POST_131; + if (!MemoryPoW.verify2(data, powBufferSize, powDifficulty, nonce)) { LOGGER.debug(() -> String.format("Peer %s sent incorrect RESPONSE nonce", peer)); return null; } @@ -194,7 +214,9 @@ public void action(Peer peer) { // No point computing for dead peer return; - Integer nonce = MemoryPoW.compute2(data, POW_BUFFER_SIZE, POW_DIFFICULTY); + int powBufferSize = peer.getPeersVersion() < PEER_VERSION_131 ? POW_BUFFER_SIZE_PRE_131 : POW_BUFFER_SIZE_POST_131; + int powDifficulty = peer.getPeersVersion() < PEER_VERSION_131 ? POW_DIFFICULTY_PRE_131 : POW_DIFFICULTY_POST_131; + Integer nonce = MemoryPoW.compute2(data, powBufferSize, powDifficulty); Message responseMessage = new ResponseMessage(nonce, data); if (!peer.sendMessage(responseMessage)) @@ -240,10 +262,18 @@ public void action(Peer peer) { /** Maximum allowed difference between peer's reported timestamp and when they connected, in milliseconds. */ private static final long MAX_TIMESTAMP_DELTA = 30 * 1000L; // ms - private static final Pattern VERSION_PATTERN = Pattern.compile(Controller.VERSION_PREFIX + "(\\d{1,3})\\.(\\d{1,5})\\.(\\d{1,5})"); + private static final long PEER_VERSION_131 = 0x0100030001L; + + /** Minimum peer version that we are allowed to communicate with */ + private static final String MIN_PEER_VERSION = "3.1.0"; + + private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes + private static final int POW_DIFFICULTY_PRE_131 = 8; // leading zero bits + // Can always be made harder in the future... + private static final int POW_BUFFER_SIZE_POST_131 = 2 * 1024 * 1024; // bytes + private static final int POW_DIFFICULTY_POST_131 = 2; // leading zero bits + - private static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes - private static final int POW_DIFFICULTY = 8; // leading zero bits private static final ExecutorService responseExecutor = Executors.newFixedThreadPool(Settings.getInstance().getNetworkPoWComputePoolSize(), new DaemonThreadFactory("Network-PoW")); private static final byte[] ZERO_CHALLENGE = new byte[ChallengeMessage.CHALLENGE_LENGTH]; diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 9866697b5..6a5177ad4 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -1,1162 +1,1493 @@ package org.qortal.network; -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.StandardSocketOptions; -import java.net.UnknownHostException; -import java.nio.channels.CancelledKeyException; -import java.nio.channels.SelectionKey; -import java.nio.channels.Selector; -import java.nio.channels.ServerSocketChannel; -import java.nio.channels.SocketChannel; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - +import com.dosse.upnp.UPnP; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters; import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters; import org.qortal.block.BlockChain; import org.qortal.controller.Controller; +import org.qortal.controller.arbitrary.ArbitraryDataFileListManager; import org.qortal.crypto.Crypto; import org.qortal.data.block.BlockData; import org.qortal.data.network.PeerData; import org.qortal.data.transaction.TransactionData; -import org.qortal.network.message.GetPeersMessage; -import org.qortal.network.message.GetUnconfirmedTransactionsMessage; -import org.qortal.network.message.HeightV2Message; -import org.qortal.network.message.Message; -import org.qortal.network.message.PeersV2Message; -import org.qortal.network.message.PingMessage; -import org.qortal.network.message.TransactionSignaturesMessage; +import org.qortal.network.message.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; -import org.qortal.transform.Transformer; +import org.qortal.utils.Base58; import org.qortal.utils.ExecuteProduceConsume; -// import org.qortal.utils.ExecutorDumper; import org.qortal.utils.ExecuteProduceConsume.StatsSnapshot; import org.qortal.utils.NTP; import org.qortal.utils.NamedThreadFactory; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.StandardSocketOptions; +import java.net.UnknownHostException; +import java.nio.channels.*; +import java.security.SecureRandom; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + // For managing peers public class Network { - - private static final Logger LOGGER = LogManager.getLogger(Network.class); - private static Network instance; - - private static final int LISTEN_BACKLOG = 10; - /** How long before retrying after a connection failure, in milliseconds. */ - private static final long CONNECT_FAILURE_BACKOFF = 5 * 60 * 1000L; // ms - /** How long between informational broadcasts to all connected peers, in milliseconds. */ - private static final long BROADCAST_INTERVAL = 60 * 1000L; // ms - /** Maximum time since last successful connection for peer info to be propagated, in milliseconds. */ - private static final long RECENT_CONNECTION_THRESHOLD = 24 * 60 * 60 * 1000L; // ms - /** Maximum time since last connection attempt before a peer is potentially considered "old", in milliseconds. */ - private static final long OLD_PEER_ATTEMPTED_PERIOD = 24 * 60 * 60 * 1000L; // ms - /** Maximum time since last successful connection before a peer is potentially considered "old", in milliseconds. */ - private static final long OLD_PEER_CONNECTION_PERIOD = 7 * 24 * 60 * 60 * 1000L; // ms - /** Maximum time allowed for handshake to complete, in milliseconds. */ - private static final long HANDSHAKE_TIMEOUT = 60 * 1000L; // ms - - private static final byte[] MAINNET_MESSAGE_MAGIC = new byte[] { 0x51, 0x4f, 0x52, 0x54 }; // QORT - private static final byte[] TESTNET_MESSAGE_MAGIC = new byte[] { 0x71, 0x6f, 0x72, 0x54 }; // qorT - - private static final String[] INITIAL_PEERS = new String[] { - "node1.qortal.org", - "node2.qortal.org", - "node3.qortal.org", - "node4.qortal.org", - "node5.qortal.org", - "node6.qortal.org", - "node7.qortal.org", - "node8.qortal.org", - "node9.qortal.org", - "node10.qortal.org" - }; - - private static final long NETWORK_EPC_KEEPALIVE = 10L; // seconds - - public static final int MAX_SIGNATURES_PER_REPLY = 500; - public static final int MAX_BLOCK_SUMMARIES_PER_REPLY = 500; - - private final Ed25519PrivateKeyParameters edPrivateKeyParams; - private final Ed25519PublicKeyParameters edPublicKeyParams; - private final String ourNodeId; - - private final int maxMessageSize; - - private List allKnownPeers; - private List connectedPeers; - private List selfPeers; - - private ExecuteProduceConsume networkEPC; - private Selector channelSelector; - private ServerSocketChannel serverChannel; - private Iterator channelIterator = null; - - private int minOutboundPeers; - private int maxPeers; - private long nextConnectTaskTimestamp; - - private ExecutorService broadcastExecutor; - private long nextBroadcastTimestamp; - - private Lock mergePeersLock; - - // Constructors - - private Network() { - connectedPeers = new ArrayList<>(); - selfPeers = new ArrayList<>(); - - // Generate our ID - byte[] seed = new byte[Transformer.PRIVATE_KEY_LENGTH]; - new SecureRandom().nextBytes(seed); - - edPrivateKeyParams = new Ed25519PrivateKeyParameters(seed, 0); - edPublicKeyParams = edPrivateKeyParams.generatePublicKey(); - ourNodeId = Crypto.toNodeAddress(edPublicKeyParams.getEncoded()); - - maxMessageSize = 4 + 1 + 4 + BlockChain.getInstance().getMaxBlockSize(); - - minOutboundPeers = Settings.getInstance().getMinOutboundPeers(); - maxPeers = Settings.getInstance().getMaxPeers(); - - nextConnectTaskTimestamp = 0; // First connect once NTP syncs - - broadcastExecutor = Executors.newCachedThreadPool(); - nextBroadcastTimestamp = 0; // First broadcast once NTP syncs - - mergePeersLock = new ReentrantLock(); - - // We'll use a cached thread pool but with more aggressive timeout. - ExecutorService networkExecutor = new ThreadPoolExecutor(1, - Settings.getInstance().getMaxNetworkThreadPoolSize(), - NETWORK_EPC_KEEPALIVE, TimeUnit.SECONDS, - new SynchronousQueue(), - new NamedThreadFactory("Network-EPC")); - networkEPC = new NetworkProcessor(networkExecutor); - } - - public void start() throws IOException, DataException { - // Grab P2P port from settings - int listenPort = Settings.getInstance().getListenPort(); - - // Grab P2P bind address from settings - try { - InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress()); - InetSocketAddress endpoint = new InetSocketAddress(bindAddr, listenPort); - - channelSelector = Selector.open(); - - // Set up listen socket - serverChannel = ServerSocketChannel.open(); - serverChannel.configureBlocking(false); - serverChannel.setOption(StandardSocketOptions.SO_REUSEADDR, true); - serverChannel.bind(endpoint, LISTEN_BACKLOG); - serverChannel.register(channelSelector, SelectionKey.OP_ACCEPT); - } catch (UnknownHostException e) { - LOGGER.error(String.format("Can't bind listen socket to address %s", Settings.getInstance().getBindAddress())); - throw new IOException("Can't bind listen socket to address", e); - } catch (IOException e) { - LOGGER.error(String.format("Can't create listen socket: %s", e.getMessage())); - throw new IOException("Can't create listen socket", e); - } - - // Load all known peers from repository - try (final Repository repository = RepositoryManager.getRepository()) { - allKnownPeers = repository.getNetworkRepository().getAllPeers(); - } - - // Start up first networking thread - networkEPC.start(); - } - - // Getters / setters - - public static synchronized Network getInstance() { - if (instance == null) - instance = new Network(); - - return instance; - } - - public byte[] getMessageMagic() { - return Settings.getInstance().isTestNet() ? TESTNET_MESSAGE_MAGIC : MAINNET_MESSAGE_MAGIC; - } - - public String getOurNodeId() { - return this.ourNodeId; - } - - /*package*/ byte[] getOurPublicKey() { - return this.edPublicKeyParams.getEncoded(); - } - - /** Maximum message size (bytes). Needs to be at least maximum block size + MAGIC + message type, etc. */ - /* package */ int getMaxMessageSize() { - return this.maxMessageSize; - } - - public StatsSnapshot getStatsSnapshot() { - return this.networkEPC.getStatsSnapshot(); - } - - // Peer lists - - public List getAllKnownPeers() { - synchronized (this.allKnownPeers) { - return new ArrayList<>(this.allKnownPeers); - } - } - - public List getConnectedPeers() { - synchronized (this.connectedPeers) { - return new ArrayList<>(this.connectedPeers); - } - } - - public List getSelfPeers() { - synchronized (this.selfPeers) { - return new ArrayList<>(this.selfPeers); - } - } - - /** Returns list of connected peers that have completed handshaking. */ - public List getHandshakedPeers() { - synchronized (this.connectedPeers) { - return this.connectedPeers.stream().filter(peer -> peer.getHandshakeStatus() == Handshake.COMPLETED).collect(Collectors.toList()); - } - } - - /** Returns list of peers we connected to that have completed handshaking. */ - public List getOutboundHandshakedPeers() { - synchronized (this.connectedPeers) { - return this.connectedPeers.stream().filter(peer -> peer.isOutbound() && peer.getHandshakeStatus() == Handshake.COMPLETED).collect(Collectors.toList()); - } - } - - /** Returns first peer that has completed handshaking and has matching public key. */ - public Peer getHandshakedPeerWithPublicKey(byte[] publicKey) { - synchronized (this.connectedPeers) { - return this.connectedPeers.stream().filter(peer -> peer.getHandshakeStatus() == Handshake.COMPLETED && Arrays.equals(peer.getPeersPublicKey(), publicKey)).findFirst().orElse(null); - } - } - - // Peer list filters - - /** Must be inside synchronized (this.selfPeers) {...} */ - private final Predicate isSelfPeer = peerData -> { - PeerAddress peerAddress = peerData.getAddress(); - return this.selfPeers.stream().anyMatch(selfPeer -> selfPeer.equals(peerAddress)); - }; - - /** Must be inside synchronized (this.connectedPeers) {...} */ - private final Predicate isConnectedPeer = peerData -> { - PeerAddress peerAddress = peerData.getAddress(); - return this.connectedPeers.stream().anyMatch(peer -> peer.getPeerData().getAddress().equals(peerAddress)); - }; - - /** Must be inside synchronized (this.connectedPeers) {...} */ - private final Predicate isResolvedAsConnectedPeer = peerData -> { - try { - InetSocketAddress resolvedSocketAddress = peerData.getAddress().toSocketAddress(); - return this.connectedPeers.stream().anyMatch(peer -> peer.getResolvedAddress().equals(resolvedSocketAddress)); - } catch (UnknownHostException e) { - // Can't resolve - no point even trying to connect - return true; - } - }; - - // Initial setup - - public static void installInitialPeers(Repository repository) throws DataException { - for (String address : INITIAL_PEERS) { - PeerAddress peerAddress = PeerAddress.fromString(address); - - PeerData peerData = new PeerData(peerAddress, System.currentTimeMillis(), "INIT"); - repository.getNetworkRepository().save(peerData); - } - - repository.saveChanges(); - } - - // Main thread - - class NetworkProcessor extends ExecuteProduceConsume { - - public NetworkProcessor(ExecutorService executor) { - super(executor); - } - - @Override - protected void onSpawnFailure() { - // For debugging: - // ExecutorDumper.dump(this.executor, 3, ExecuteProduceConsume.class); - } - - @Override - protected Task produceTask(boolean canBlock) throws InterruptedException { - Task task; - - task = maybeProducePeerMessageTask(); - if (task != null) - return task; - - final Long now = NTP.getTime(); - - task = maybeProducePeerPingTask(now); - if (task != null) - return task; - - task = maybeProduceConnectPeerTask(now); - if (task != null) - return task; - - task = maybeProduceBroadcastTask(now); - if (task != null) - return task; - - // Only this method can block to reduce CPU spin - task = maybeProduceChannelTask(canBlock); - if (task != null) - return task; - - // Really nothing to do - return null; - } - - private Task maybeProducePeerMessageTask() { - for (Peer peer : getConnectedPeers()) { - Task peerTask = peer.getMessageTask(); - if (peerTask != null) - return peerTask; - } - - return null; - } - - private Task maybeProducePeerPingTask(Long now) { - // Ask connected peers whether they need a ping - for (Peer peer : getHandshakedPeers()) { - Task peerTask = peer.getPingTask(now); - if (peerTask != null) - return peerTask; - } - - return null; - } - - class PeerConnectTask implements ExecuteProduceConsume.Task { - private final Peer peer; - - public PeerConnectTask(Peer peer) { - this.peer = peer; - } - - @Override - public void perform() throws InterruptedException { - connectPeer(peer); - } - } - - private Task maybeProduceConnectPeerTask(Long now) throws InterruptedException { - if (now == null || now < nextConnectTaskTimestamp) - return null; - - if (getOutboundHandshakedPeers().size() >= minOutboundPeers) - return null; - - nextConnectTaskTimestamp = now + 1000L; - - Peer targetPeer = getConnectablePeer(now); - if (targetPeer == null) - return null; - - // Create connection task - return new PeerConnectTask(targetPeer); - } - - private Task maybeProduceBroadcastTask(Long now) { - if (now == null || now < nextBroadcastTimestamp) - return null; - - nextBroadcastTimestamp = now + BROADCAST_INTERVAL; - return () -> Controller.getInstance().doNetworkBroadcast(); - } - - class ChannelTask implements ExecuteProduceConsume.Task { - private final SelectionKey selectionKey; - - public ChannelTask(SelectionKey selectionKey) { - this.selectionKey = selectionKey; - } - - @Override - public void perform() throws InterruptedException { - try { - LOGGER.trace(() -> String.format("Thread %d has pending channel: %s, with ops %d", - Thread.currentThread().getId(), selectionKey.channel(), selectionKey.readyOps())); - - // process pending channel task - if (selectionKey.isReadable()) { - connectionRead((SocketChannel) selectionKey.channel()); - } else if (selectionKey.isAcceptable()) { - acceptConnection((ServerSocketChannel) selectionKey.channel()); - } - - LOGGER.trace(() -> String.format("Thread %d processed channel: %s", Thread.currentThread().getId(), selectionKey.channel())); - } catch (CancelledKeyException e) { - LOGGER.trace(() -> String.format("Thread %s encountered cancelled channel: %s", Thread.currentThread().getId(), selectionKey.channel())); - } - } - - private void connectionRead(SocketChannel socketChannel) { - Peer peer = getPeerFromChannel(socketChannel); - if (peer == null) - return; - - try { - peer.readChannel(); - } catch (IOException e) { - if (e.getMessage() != null && e.getMessage().toLowerCase().contains("onnection reset")) { - peer.disconnect("Connection reset"); - return; - } - - LOGGER.trace(() -> String.format("Network thread %s encountered I/O error: %s", Thread.currentThread().getId(), e.getMessage()), e); - peer.disconnect("I/O error"); - } - } - } - - private Task maybeProduceChannelTask(boolean canBlock) throws InterruptedException { - final SelectionKey nextSelectionKey; - - // anything to do? - if (channelIterator == null) { - try { - if (canBlock) - channelSelector.select(1000L); - else - channelSelector.selectNow(); - } catch (IOException e) { - LOGGER.warn(String.format("Channel selection threw IOException: %s", e.getMessage())); - return null; - } - - if (Thread.currentThread().isInterrupted()) - throw new InterruptedException(); - - channelIterator = channelSelector.selectedKeys().iterator(); - } - - if (channelIterator.hasNext()) { - nextSelectionKey = channelIterator.next(); - channelIterator.remove(); - } else { - nextSelectionKey = null; - channelIterator = null; // Nothing to do so reset iterator to cause new select - } - - LOGGER.trace(() -> String.format("Thread %d, nextSelectionKey %s, channelIterator now %s", - Thread.currentThread().getId(), nextSelectionKey, channelIterator)); - - if (nextSelectionKey == null) - return null; - - return new ChannelTask(nextSelectionKey); - } - } - - private void acceptConnection(ServerSocketChannel serverSocketChannel) throws InterruptedException { - SocketChannel socketChannel; - - try { - socketChannel = serverSocketChannel.accept(); - } catch (IOException e) { - return; - } - - // No connection actually accepted? - if (socketChannel == null) - return; - - final Long now = NTP.getTime(); - Peer newPeer; - - try { - if (now == null) { - LOGGER.debug(() -> String.format("Connection discarded from peer %s due to lack of NTP sync", PeerAddress.fromSocket(socketChannel.socket()))); - socketChannel.close(); - return; - } - - synchronized (this.connectedPeers) { - if (connectedPeers.size() >= maxPeers) { - // We have enough peers - LOGGER.debug(() -> String.format("Connection discarded from peer %s", PeerAddress.fromSocket(socketChannel.socket()))); - socketChannel.close(); - return; - } - - LOGGER.debug(() -> String.format("Connection accepted from peer %s", PeerAddress.fromSocket(socketChannel.socket()))); - - newPeer = new Peer(socketChannel, channelSelector); - this.connectedPeers.add(newPeer); - } - } catch (IOException e) { - if (socketChannel.isOpen()) - try { - socketChannel.close(); - } catch (IOException ce) { - // Couldn't close? - } - - return; - } - - this.onPeerReady(newPeer); - } - - private Peer getConnectablePeer(final Long now) throws InterruptedException { - // We can't block here so use tryRepository(). We don't NEED to connect a new peer. - try (final Repository repository = RepositoryManager.tryRepository()) { - if (repository == null) - return null; - - // Find an address to connect to - List peers = this.getAllKnownPeers(); - - // Don't consider peers with recent connection failures - final long lastAttemptedThreshold = now - CONNECT_FAILURE_BACKOFF; - peers.removeIf(peerData -> peerData.getLastAttempted() != null && - (peerData.getLastConnected() == null || peerData.getLastConnected() < peerData.getLastAttempted()) && - peerData.getLastAttempted() > lastAttemptedThreshold); - - // Don't consider peers that we know loop back to ourself - synchronized (this.selfPeers) { - peers.removeIf(isSelfPeer); - } - - synchronized (this.connectedPeers) { - // Don't consider already connected peers (simple address match) - peers.removeIf(isConnectedPeer); - - // Don't consider already connected peers (resolved address match) - // XXX This might be too slow if we end up waiting a long time for hostnames to resolve via DNS - peers.removeIf(isResolvedAsConnectedPeer); - } - - // Any left? - if (peers.isEmpty()) - return null; - - // Pick random peer - int peerIndex = new Random().nextInt(peers.size()); - - // Pick candidate - PeerData peerData = peers.get(peerIndex); - Peer newPeer = new Peer(peerData); - - // Update connection attempt info - peerData.setLastAttempted(now); - synchronized (this.allKnownPeers) { - repository.getNetworkRepository().save(peerData); - repository.saveChanges(); - } - - return newPeer; - } catch (DataException e) { - LOGGER.error("Repository issue while finding a connectable peer", e); - return null; - } - } - - private void connectPeer(Peer newPeer) throws InterruptedException { - SocketChannel socketChannel = newPeer.connect(this.channelSelector); - if (socketChannel == null) - return; - - if (Thread.currentThread().isInterrupted()) - return; - - synchronized (this.connectedPeers) { - this.connectedPeers.add(newPeer); - } - - this.onPeerReady(newPeer); - } - - private Peer getPeerFromChannel(SocketChannel socketChannel) { - synchronized (this.connectedPeers) { - for (Peer peer : this.connectedPeers) - if (peer.getSocketChannel() == socketChannel) - return peer; - } - - return null; - } - - // Peer callbacks - - /*package*/ void wakeupChannelSelector() { - this.channelSelector.wakeup(); - } - - /*package*/ boolean verify(byte[] signature, byte[] message) { - return Crypto.verify(this.edPublicKeyParams.getEncoded(), signature, message); - } - - /*package*/ byte[] sign(byte[] message) { - return Crypto.sign(this.edPrivateKeyParams, message); - } - - /*package*/ byte[] getSharedSecret(byte[] publicKey) { - return Crypto.getSharedSecret(this.edPrivateKeyParams.getEncoded(), publicKey); - } - - /** Called when Peer's thread has setup and is ready to process messages */ - public void onPeerReady(Peer peer) { - onHandshakingMessage(peer, null, Handshake.STARTED); - } - - public void onDisconnect(Peer peer) { - // Notify Controller - Controller.getInstance().onPeerDisconnect(peer); - - synchronized (this.connectedPeers) { - this.connectedPeers.remove(peer); - } - } - - public void peerMisbehaved(Peer peer) { - PeerData peerData = peer.getPeerData(); - peerData.setLastMisbehaved(NTP.getTime()); - - // Only update repository if outbound peer - if (peer.isOutbound()) - try (final Repository repository = RepositoryManager.getRepository()) { - synchronized (this.allKnownPeers) { - repository.getNetworkRepository().save(peerData); - repository.saveChanges(); - } - } catch (DataException e) { - LOGGER.warn("Repository issue while updating peer synchronization info", e); - } - } - - /** Called when a new message arrives for a peer. message can be null if called after connection */ - public void onMessage(Peer peer, Message message) { - if (message != null) - LOGGER.trace(() -> String.format("Processing %s message with ID %d from peer %s", message.getType().name(), message.getId(), peer)); - - Handshake handshakeStatus = peer.getHandshakeStatus(); - if (handshakeStatus != Handshake.COMPLETED) { - onHandshakingMessage(peer, message, handshakeStatus); - return; - } - - // Should be non-handshaking messages from now on - - // Ordered by message type value - switch (message.getType()) { - case GET_PEERS: - onGetPeersMessage(peer, message); - break; - - case PING: - onPingMessage(peer, message); - break; - - case HELLO: - case CHALLENGE: - case RESPONSE: - LOGGER.debug(() -> String.format("Unexpected handshaking message %s from peer %s", message.getType().name(), peer)); - peer.disconnect("unexpected handshaking message"); - return; - - case PEERS_V2: - onPeersV2Message(peer, message); - break; - - default: - // Bump up to controller for possible action - Controller.getInstance().onNetworkMessage(peer, message); - break; - } - } - - private void onHandshakingMessage(Peer peer, Message message, Handshake handshakeStatus) { - try { - // Still handshaking - LOGGER.trace(() -> String.format("Handshake status %s, message %s from peer %s", handshakeStatus.name(), (message != null ? message.getType().name() : "null"), peer)); - - // Check message type is as expected - if (handshakeStatus.expectedMessageType != null && message.getType() != handshakeStatus.expectedMessageType) { - LOGGER.debug(() -> String.format("Unexpected %s message from %s, expected %s", message.getType().name(), peer, handshakeStatus.expectedMessageType)); - peer.disconnect("unexpected message"); - return; - } - - Handshake newHandshakeStatus = handshakeStatus.onMessage(peer, message); - - if (newHandshakeStatus == null) { - // Handshake failure - LOGGER.debug(() -> String.format("Handshake failure with peer %s message %s", peer, message.getType().name())); - peer.disconnect("handshake failure"); - return; - } - - if (peer.isOutbound()) - // If we made outbound connection then we need to act first - newHandshakeStatus.action(peer); - else - // We have inbound connection so we need to respond in kind with what we just received - handshakeStatus.action(peer); - - peer.setHandshakeStatus(newHandshakeStatus); - - if (newHandshakeStatus == Handshake.COMPLETED) - this.onHandshakeCompleted(peer); - } finally { - peer.resetHandshakeMessagePending(); - } - } - - private void onGetPeersMessage(Peer peer, Message message) { - // Send our known peers - if (!peer.sendMessage(this.buildPeersMessage(peer))) - peer.disconnect("failed to send peers list"); - } - - private void onPingMessage(Peer peer, Message message) { - PingMessage pingMessage = (PingMessage) message; - - // Generate 'pong' using same ID - PingMessage pongMessage = new PingMessage(); - pongMessage.setId(pingMessage.getId()); - - if (!peer.sendMessage(pongMessage)) - peer.disconnect("failed to send ping reply"); - } - - private void onPeersV2Message(Peer peer, Message message) { - PeersV2Message peersV2Message = (PeersV2Message) message; - - List peerV2Addresses = peersV2Message.getPeerAddresses(); - - // First entry contains remote peer's listen port but empty address. - int peerPort = peerV2Addresses.get(0).getPort(); - peerV2Addresses.remove(0); - - // If inbound peer, use listen port and socket address to recreate first entry - if (!peer.isOutbound()) { - PeerAddress sendingPeerAddress = PeerAddress.fromString(peer.getPeerData().getAddress().getHost() + ":" + peerPort); - LOGGER.trace(() -> String.format("PEERS_V2 sending peer's listen address: %s", sendingPeerAddress.toString())); - peerV2Addresses.add(0, sendingPeerAddress); - } - - opportunisticMergePeers(peer.toString(), peerV2Addresses); - } - - /*pacakge*/ void onHandshakeCompleted(Peer peer) { - LOGGER.debug(String.format("Handshake completed with peer %s", peer)); - - // Are we already connected to this peer? - Peer existingPeer = getHandshakedPeerWithPublicKey(peer.getPeersPublicKey()); - // NOTE: actual object reference compare, not Peer.equals() - if (existingPeer != peer) { - LOGGER.info(() -> String.format("We already have a connection with peer %s - discarding", peer)); - peer.disconnect("existing connection"); - return; - } - - // Make a note that we've successfully completed handshake (and when) - peer.getPeerData().setLastConnected(NTP.getTime()); - - // Update connection info for outbound peers only - if (peer.isOutbound()) - try (final Repository repository = RepositoryManager.getRepository()) { - synchronized (this.allKnownPeers) { - repository.getNetworkRepository().save(peer.getPeerData()); - repository.saveChanges(); - } - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while trying to update outbound peer %s", peer), e); - } - - // Start regular pings - peer.startPings(); - - // Only the outbound side needs to send anything (after we've received handshake-completing response). - // (If inbound sent anything here, it's possible it could be processed out-of-order with handshake message). - - if (peer.isOutbound()) { - // Send our height - Message heightMessage = buildHeightMessage(peer, Controller.getInstance().getChainTip()); - if (!peer.sendMessage(heightMessage)) { - peer.disconnect("failed to send height/info"); - return; - } - - // Send our peers list - Message peersMessage = this.buildPeersMessage(peer); - if (!peer.sendMessage(peersMessage)) - peer.disconnect("failed to send peers list"); - - // Request their peers list - Message getPeersMessage = new GetPeersMessage(); - if (!peer.sendMessage(getPeersMessage)) - peer.disconnect("failed to request peers list"); - } - - // Ask Controller if they want to do anything - Controller.getInstance().onPeerHandshakeCompleted(peer); - } - - // Message-building calls - - /** Returns PEERS message made from peers we've connected to recently, and this node's details */ - public Message buildPeersMessage(Peer peer) { - List knownPeers = this.getAllKnownPeers(); - - // Filter out peers that we've not connected to ever or within X milliseconds - final long connectionThreshold = NTP.getTime() - RECENT_CONNECTION_THRESHOLD; - Predicate notRecentlyConnected = peerData -> { - final Long lastAttempted = peerData.getLastAttempted(); - final Long lastConnected = peerData.getLastConnected(); - - if (lastAttempted == null || lastConnected == null) - return true; - - if (lastConnected < lastAttempted) - return true; - - if (lastConnected < connectionThreshold) - return true; - - return false; - }; - knownPeers.removeIf(notRecentlyConnected); - - List peerAddresses = new ArrayList<>(); - - for (PeerData peerData : knownPeers) { - try { - InetAddress address = InetAddress.getByName(peerData.getAddress().getHost()); - - // Don't send 'local' addresses if peer is not 'local'. e.g. don't send localhost:9084 to node4.qortal.org - if (!peer.isLocal() && Peer.isAddressLocal(address)) - continue; - - peerAddresses.add(peerData.getAddress()); - } catch (UnknownHostException e) { - // Couldn't resolve hostname to IP address so discard - } - } - - // New format PEERS_V2 message that supports hostnames, IPv6 and ports - return new PeersV2Message(peerAddresses); - } - - public Message buildHeightMessage(Peer peer, BlockData blockData) { - // HEIGHT_V2 contains way more useful info - return new HeightV2Message(blockData.getHeight(), blockData.getSignature(), blockData.getTimestamp(), blockData.getMinterPublicKey()); - } - - public Message buildNewTransactionMessage(Peer peer, TransactionData transactionData) { - // In V2 we send out transaction signature only and peers can decide whether to request the full transaction - return new TransactionSignaturesMessage(Collections.singletonList(transactionData.getSignature())); - } - - public Message buildGetUnconfirmedTransactionsMessage(Peer peer) { - return new GetUnconfirmedTransactionsMessage(); - } - - // Peer-management calls - - public void noteToSelf(Peer peer) { - LOGGER.info(() -> String.format("No longer considering peer address %s as it connects to self", peer)); - - synchronized (this.selfPeers) { - this.selfPeers.add(peer.getPeerData().getAddress()); - } - } - - public boolean forgetPeer(PeerAddress peerAddress) throws DataException { - int numDeleted; - - synchronized (this.allKnownPeers) { - this.allKnownPeers.removeIf(peerData -> peerData.getAddress().equals(peerAddress)); - - try (final Repository repository = RepositoryManager.getRepository()) { - numDeleted = repository.getNetworkRepository().delete(peerAddress); - repository.saveChanges(); - } - } - - disconnectPeer(peerAddress); - - return numDeleted != 0; - } - - public int forgetAllPeers() throws DataException { - int numDeleted; - - synchronized (this.allKnownPeers) { - this.allKnownPeers.clear(); - - try (final Repository repository = RepositoryManager.getRepository()) { - numDeleted = repository.getNetworkRepository().deleteAllPeers(); - repository.saveChanges(); - } - } - - for (Peer peer : this.getConnectedPeers()) - peer.disconnect("to be forgotten"); - - return numDeleted; - } - - private void disconnectPeer(PeerAddress peerAddress) { - // Disconnect peer - try { - InetSocketAddress knownAddress = peerAddress.toSocketAddress(); - - List peers = this.getConnectedPeers(); - peers.removeIf(peer -> !Peer.addressEquals(knownAddress, peer.getResolvedAddress())); - - for (Peer peer : peers) - peer.disconnect("to be forgotten"); - } catch (UnknownHostException e) { - // Unknown host isn't going to match any of our connected peers so ignore - } - } - - // Network-wide calls - - public void prunePeers() throws DataException { - final Long now = NTP.getTime(); - if (now == null) - return; - - // Disconnect peers that are stuck during handshake - List handshakePeers = this.getConnectedPeers(); - - // Disregard peers that have completed handshake or only connected recently - handshakePeers.removeIf(peer -> peer.getHandshakeStatus() == Handshake.COMPLETED || peer.getConnectionTimestamp() == null || peer.getConnectionTimestamp() > now - HANDSHAKE_TIMEOUT); - - for (Peer peer : handshakePeers) - peer.disconnect(String.format("handshake timeout at %s", peer.getHandshakeStatus().name())); - - // Prune 'old' peers from repository... - // Pruning peers isn't critical so no need to block for a repository instance. - try (final Repository repository = RepositoryManager.tryRepository()) { - if (repository == null) - return; - - synchronized (this.allKnownPeers) { - // Fetch all known peers - List peers = new ArrayList<>(this.allKnownPeers); - - // 'Old' peers: - // We attempted to connect within the last day - // but we last managed to connect over a week ago. - Predicate isNotOldPeer = peerData -> { - if (peerData.getLastAttempted() == null || peerData.getLastAttempted() < now - OLD_PEER_ATTEMPTED_PERIOD) - return true; - - if (peerData.getLastConnected() == null || peerData.getLastConnected() > now - OLD_PEER_CONNECTION_PERIOD) - return true; - - return false; - }; - - // Disregard peers that are NOT 'old' - peers.removeIf(isNotOldPeer); - - // Don't consider already connected peers (simple address match) - synchronized (this.connectedPeers) { - peers.removeIf(isConnectedPeer); - } - - for (PeerData peerData : peers) { - LOGGER.debug(() -> String.format("Deleting old peer %s from repository", peerData.getAddress().toString())); - repository.getNetworkRepository().delete(peerData.getAddress()); - - // Delete from known peer cache too - this.allKnownPeers.remove(peerData); - } - - repository.saveChanges(); - } - } - } - - public boolean mergePeers(String addedBy, long addedWhen, List peerAddresses) throws DataException { - mergePeersLock.lock(); - - try (final Repository repository = RepositoryManager.getRepository()) { - return this.mergePeers(repository, addedBy, addedWhen, peerAddresses); - } finally { - mergePeersLock.unlock(); - } - } - - private void opportunisticMergePeers(String addedBy, List peerAddresses) { - final Long addedWhen = NTP.getTime(); - if (addedWhen == null) - return; - - // Serialize using lock to prevent repository deadlocks - if (!mergePeersLock.tryLock()) - return; - - try { - // Merging peers isn't critical so don't block for a repository instance. - try (final Repository repository = RepositoryManager.tryRepository()) { - if (repository == null) - return; - - this.mergePeers(repository, addedBy, addedWhen, peerAddresses); - - } catch (DataException e) { - // Already logged by this.mergePeers() - } - } finally { - mergePeersLock.unlock(); - } - } - - private boolean mergePeers(Repository repository, String addedBy, long addedWhen, List peerAddresses) throws DataException { - List newPeers; - synchronized (this.allKnownPeers) { - for (PeerData knownPeerData : this.allKnownPeers) { - // Filter out duplicates, without resolving via DNS - Predicate isKnownAddress = peerAddress -> knownPeerData.getAddress().equals(peerAddress); - peerAddresses.removeIf(isKnownAddress); - } - - if (peerAddresses.isEmpty()) - return false; - - // Add leftover peer addresses to known peers list - newPeers = peerAddresses.stream().map(peerAddress -> new PeerData(peerAddress, addedWhen, addedBy)).collect(Collectors.toList()); - - this.allKnownPeers.addAll(newPeers); - - try { - // Save new peers into database - for (PeerData peerData : newPeers) { - LOGGER.info(String.format("Adding new peer %s to repository", peerData.getAddress())); - repository.getNetworkRepository().save(peerData); - } - - repository.saveChanges(); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while merging peers list from %s", addedBy), e); - throw e; - } - - return true; - } - } - - public void broadcast(Function peerMessageBuilder) { - class Broadcaster implements Runnable { - private final Random random = new Random(); - - private List targetPeers; - private Function peerMessageBuilder; - - public Broadcaster(List targetPeers, Function peerMessageBuilder) { - this.targetPeers = targetPeers; - this.peerMessageBuilder = peerMessageBuilder; - } - - @Override - public void run() { - Thread.currentThread().setName("Network Broadcast"); - - for (Peer peer : targetPeers) { - // Very short sleep to reduce strain, improve multi-threading and catch interrupts - try { - Thread.sleep(random.nextInt(20) + 20L); - } catch (InterruptedException e) { - break; - } - - Message message = peerMessageBuilder.apply(peer); - - if (message == null) - continue; - - if (!peer.sendMessage(message)) - peer.disconnect("failed to broadcast message"); - } - - Thread.currentThread().setName("Network Broadcast (dormant)"); - } - } - - try { - broadcastExecutor.execute(new Broadcaster(this.getHandshakedPeers(), peerMessageBuilder)); - } catch (RejectedExecutionException e) { - // Can't execute - probably because we're shutting down, so ignore - } - } - - // Shutdown - - public void shutdown() { - // Close listen socket to prevent more incoming connections - if (this.serverChannel.isOpen()) - try { - this.serverChannel.close(); - } catch (IOException e) { - // Not important - } - - // Stop processing threads - try { - if (!this.networkEPC.shutdown(5000)) - LOGGER.warn("Network threads failed to terminate"); - } catch (InterruptedException e) { - LOGGER.warn("Interrupted while waiting for networking threads to terminate"); - } - - // Stop broadcasts - this.broadcastExecutor.shutdownNow(); - try { - if (!this.broadcastExecutor.awaitTermination(1000, TimeUnit.MILLISECONDS)) - LOGGER.warn("Broadcast threads failed to terminate"); - } catch (InterruptedException e) { - LOGGER.warn("Interrupted while waiting for broadcast threads failed to terminate"); - } - - // Close all peer connections - for (Peer peer : this.getConnectedPeers()) - peer.shutdown(); - } + private static final Logger LOGGER = LogManager.getLogger(Network.class); + private static Network instance; + + private static final int LISTEN_BACKLOG = 10; + /** + * How long before retrying after a connection failure, in milliseconds. + */ + private static final long CONNECT_FAILURE_BACKOFF = 5 * 60 * 1000L; // ms + /** + * How long between informational broadcasts to all connected peers, in milliseconds. + */ + private static final long BROADCAST_INTERVAL = 60 * 1000L; // ms + /** + * Maximum time since last successful connection for peer info to be propagated, in milliseconds. + */ + private static final long RECENT_CONNECTION_THRESHOLD = 24 * 60 * 60 * 1000L; // ms + /** + * Maximum time since last connection attempt before a peer is potentially considered "old", in milliseconds. + */ + private static final long OLD_PEER_ATTEMPTED_PERIOD = 24 * 60 * 60 * 1000L; // ms + /** + * Maximum time since last successful connection before a peer is potentially considered "old", in milliseconds. + */ + private static final long OLD_PEER_CONNECTION_PERIOD = 7 * 24 * 60 * 60 * 1000L; // ms + /** + * Maximum time allowed for handshake to complete, in milliseconds. + */ + private static final long HANDSHAKE_TIMEOUT = 60 * 1000L; // ms + + private static final byte[] MAINNET_MESSAGE_MAGIC = new byte[]{0x51, 0x4f, 0x52, 0x54}; // QORT + private static final byte[] TESTNET_MESSAGE_MAGIC = new byte[]{0x71, 0x6f, 0x72, 0x54}; // qorT + + private static final String[] INITIAL_PEERS = new String[]{ + "node1.qortal.org", "node2.qortal.org", "node3.qortal.org", "node4.qortal.org", "node5.qortal.org", + "node6.qortal.org", "node7.qortal.org", "node8.qortal.org", "node9.qortal.org", "node10.qortal.org", + "node.qortal.ru", "node2.qortal.ru", "node3.qortal.ru", "node.qortal.uk", "node22.qortal.org", + "cinfu1.crowetic.com", "node.cwd.systems", "bootstrap.cwd.systems", "node1.qortalnodes.live", + "node2.qortalnodes.live", "node3.qortalnodes.live", "node4.qortalnodes.live", "node5.qortalnodes.live", + "node6.qortalnodes.live", "node7.qortalnodes.live", "node8.qortalnodes.live" + }; + + private static final long NETWORK_EPC_KEEPALIVE = 10L; // seconds + + public static final int MAX_SIGNATURES_PER_REPLY = 500; + public static final int MAX_BLOCK_SUMMARIES_PER_REPLY = 500; + + private static final long DISCONNECTION_CHECK_INTERVAL = 10 * 1000L; // milliseconds + + // Generate our node keys / ID + private final Ed25519PrivateKeyParameters edPrivateKeyParams = new Ed25519PrivateKeyParameters(new SecureRandom()); + private final Ed25519PublicKeyParameters edPublicKeyParams = edPrivateKeyParams.generatePublicKey(); + private final String ourNodeId = Crypto.toNodeAddress(edPublicKeyParams.getEncoded()); + + private final int maxMessageSize; + private final int minOutboundPeers; + private final int maxPeers; + + private long nextDisconnectionCheck = 0L; + + private final List allKnownPeers = new ArrayList<>(); + private final List connectedPeers = new ArrayList<>(); + private final List selfPeers = new ArrayList<>(); + + private final ExecuteProduceConsume networkEPC; + private Selector channelSelector; + private ServerSocketChannel serverChannel; + private Iterator channelIterator = null; + + // volatile because value is updated inside any one of the EPC threads + private volatile long nextConnectTaskTimestamp = 0L; // ms - try first connect once NTP syncs + + private final ExecutorService broadcastExecutor = Executors.newCachedThreadPool(); + // volatile because value is updated inside any one of the EPC threads + private volatile long nextBroadcastTimestamp = 0L; // ms - try first broadcast once NTP syncs + + private final Lock mergePeersLock = new ReentrantLock(); + + private List ourExternalIpAddressHistory = new ArrayList<>(); + private String ourExternalIpAddress = null; + + // Constructors + + private Network() { + maxMessageSize = 4 + 1 + 4 + BlockChain.getInstance().getMaxBlockSize(); + + minOutboundPeers = Settings.getInstance().getMinOutboundPeers(); + maxPeers = Settings.getInstance().getMaxPeers(); + + // We'll use a cached thread pool but with more aggressive timeout. + ExecutorService networkExecutor = new ThreadPoolExecutor(1, + Settings.getInstance().getMaxNetworkThreadPoolSize(), + NETWORK_EPC_KEEPALIVE, TimeUnit.SECONDS, + new SynchronousQueue(), + new NamedThreadFactory("Network-EPC")); + networkEPC = new NetworkProcessor(networkExecutor); + } + + public void start() throws IOException, DataException { + // Grab P2P port from settings + int listenPort = Settings.getInstance().getListenPort(); + + // Grab P2P bind address from settings + try { + InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress()); + InetSocketAddress endpoint = new InetSocketAddress(bindAddr, listenPort); + + channelSelector = Selector.open(); + + // Set up listen socket + serverChannel = ServerSocketChannel.open(); + serverChannel.configureBlocking(false); + serverChannel.setOption(StandardSocketOptions.SO_REUSEADDR, true); + serverChannel.bind(endpoint, LISTEN_BACKLOG); + serverChannel.register(channelSelector, SelectionKey.OP_ACCEPT); + } catch (UnknownHostException e) { + LOGGER.error("Can't bind listen socket to address {}", Settings.getInstance().getBindAddress()); + throw new IOException("Can't bind listen socket to address", e); + } catch (IOException e) { + LOGGER.error("Can't create listen socket: {}", e.getMessage()); + throw new IOException("Can't create listen socket", e); + } + + // Load all known peers from repository + synchronized (this.allKnownPeers) { List fixedNetwork = Settings.getInstance().getFixedNetwork(); + if (fixedNetwork != null && !fixedNetwork.isEmpty()) { + Long addedWhen = NTP.getTime(); + String addedBy = "fixedNetwork"; + List peerAddresses = new ArrayList<>(); + for (String address : fixedNetwork) { + PeerAddress peerAddress = PeerAddress.fromString(address); + peerAddresses.add(peerAddress); + } + List peers = peerAddresses.stream() + .map(peerAddress -> new PeerData(peerAddress, addedWhen, addedBy)) + .collect(Collectors.toList()); + this.allKnownPeers.addAll(peers); + } else { + try (Repository repository = RepositoryManager.getRepository()) { + this.allKnownPeers.addAll(repository.getNetworkRepository().getAllPeers()); + } + } + } + + // Attempt to set up UPnP. All errors are ignored. + if (Settings.getInstance().isUPnPEnabled()) { + UPnP.openPortTCP(Settings.getInstance().getListenPort()); + } + else { + UPnP.closePortTCP(Settings.getInstance().getListenPort()); + } + + // Start up first networking thread + networkEPC.start(); + } + + // Getters / setters + + public static synchronized Network getInstance() { + if (instance == null) { + instance = new Network(); + } + + return instance; + } + + public byte[] getMessageMagic() { + return Settings.getInstance().isTestNet() ? TESTNET_MESSAGE_MAGIC : MAINNET_MESSAGE_MAGIC; + } + + public String getOurNodeId() { + return this.ourNodeId; + } + + protected byte[] getOurPublicKey() { + return this.edPublicKeyParams.getEncoded(); + } + + /** + * Maximum message size (bytes). Needs to be at least maximum block size + MAGIC + message type, etc. + */ + protected int getMaxMessageSize() { + return this.maxMessageSize; + } + + public StatsSnapshot getStatsSnapshot() { + return this.networkEPC.getStatsSnapshot(); + } + + // Peer lists + + public List getAllKnownPeers() { + synchronized (this.allKnownPeers) { + return new ArrayList<>(this.allKnownPeers); + } + } + + public List getConnectedPeers() { + synchronized (this.connectedPeers) { + return new ArrayList<>(this.connectedPeers); + } + } + + public List getSelfPeers() { + synchronized (this.selfPeers) { + return new ArrayList<>(this.selfPeers); + } + } + + public boolean requestDataFromPeer(String peerAddressString, byte[] signature) { + if (peerAddressString != null) { + PeerAddress peerAddress = PeerAddress.fromString(peerAddressString); + PeerData peerData = null; + + // Reuse an existing PeerData instance if it's already in the known peers list + synchronized (this.allKnownPeers) { + peerData = this.allKnownPeers.stream() + .filter(knownPeerData -> knownPeerData.getAddress().equals(peerAddress)) + .findFirst() + .orElse(null); + } + + if (peerData == null) { + // Not a known peer, so we need to create one + Long addedWhen = NTP.getTime(); + String addedBy = "requestDataFromPeer"; + peerData = new PeerData(peerAddress, addedWhen, addedBy); + } + + if (peerData == null) { + LOGGER.info("PeerData is null when trying to request data from peer {}", peerAddressString); + return false; + } + + // Check if we're already connected to and handshaked with this peer + Peer connectedPeer = null; + synchronized (this.connectedPeers) { + connectedPeer = this.connectedPeers.stream() + .filter(p -> p.getPeerData().getAddress().equals(peerAddress)) + .findFirst() + .orElse(null); + } + boolean isConnected = (connectedPeer != null); + + boolean isHandshaked = this.getHandshakedPeers().stream() + .anyMatch(p -> p.getPeerData().getAddress().equals(peerAddress)); + + if (isConnected && isHandshaked) { + // Already connected + return this.requestDataFromConnectedPeer(connectedPeer, signature); + } + else { + // We need to connect to this peer before we can request data + try { + if (!isConnected) { + // Add this signature to the list of pending requests for this peer + LOGGER.info("Making connection to peer {} to request files for signature {}...", peerAddressString, Base58.encode(signature)); + Peer peer = new Peer(peerData); + peer.addPendingSignatureRequest(signature); + return this.connectPeer(peer); + // If connection (and handshake) is successful, data will automatically be requested + } + else if (!isHandshaked) { + LOGGER.info("Peer {} is connected but not handshaked. Not attempting a new connection.", peerAddress); + return false; + } + + } catch (InterruptedException e) { + LOGGER.info("Interrupted when connecting to peer {}", peerAddress); + return false; + } + } + } + return false; + } + + private boolean requestDataFromConnectedPeer(Peer connectedPeer, byte[] signature) { + if (signature == null) { + // Nothing to do + return false; + } + + return ArbitraryDataFileListManager.getInstance().fetchArbitraryDataFileList(connectedPeer, signature); + } + + /** + * Returns list of connected peers that have completed handshaking. + */ + public List getHandshakedPeers() { + synchronized (this.connectedPeers) { + return this.connectedPeers.stream() + .filter(peer -> peer.getHandshakeStatus() == Handshake.COMPLETED) + .collect(Collectors.toList()); + } + } + + /** + * Returns list of peers we connected to that have completed handshaking. + */ + public List getOutboundHandshakedPeers() { + synchronized (this.connectedPeers) { + return this.connectedPeers.stream() + .filter(peer -> peer.isOutbound() && peer.getHandshakeStatus() == Handshake.COMPLETED) + .collect(Collectors.toList()); + } + } + + /** + * Returns first peer that has completed handshaking and has matching public key. + */ + public Peer getHandshakedPeerWithPublicKey(byte[] publicKey) { + synchronized (this.connectedPeers) { + return this.connectedPeers.stream() + .filter(peer -> peer.getHandshakeStatus() == Handshake.COMPLETED + && Arrays.equals(peer.getPeersPublicKey(), publicKey)) + .findFirst().orElse(null); + } + } + + // Peer list filters + + /** + * Must be inside synchronized (this.selfPeers) {...} + */ + private final Predicate isSelfPeer = peerData -> { + PeerAddress peerAddress = peerData.getAddress(); + return this.selfPeers.stream().anyMatch(selfPeer -> selfPeer.equals(peerAddress)); + }; + + /** + * Must be inside synchronized (this.connectedPeers) {...} + */ + private final Predicate isConnectedPeer = peerData -> { + PeerAddress peerAddress = peerData.getAddress(); + return this.connectedPeers.stream().anyMatch(peer -> peer.getPeerData().getAddress().equals(peerAddress)); + }; + + /** + * Must be inside synchronized (this.connectedPeers) {...} + */ + private final Predicate isResolvedAsConnectedPeer = peerData -> { + try { + InetSocketAddress resolvedSocketAddress = peerData.getAddress().toSocketAddress(); + return this.connectedPeers.stream() + .anyMatch(peer -> peer.getResolvedAddress().equals(resolvedSocketAddress)); + } catch (UnknownHostException e) { + // Can't resolve - no point even trying to connect + return true; + } + }; + + // Initial setup + + public static void installInitialPeers(Repository repository) throws DataException { + for (String address : INITIAL_PEERS) { + PeerAddress peerAddress = PeerAddress.fromString(address); + + PeerData peerData = new PeerData(peerAddress, System.currentTimeMillis(), "INIT"); + repository.getNetworkRepository().save(peerData); + } + + repository.saveChanges(); + } + + // Main thread + + class NetworkProcessor extends ExecuteProduceConsume { + + NetworkProcessor(ExecutorService executor) { + super(executor); + } + + @Override + protected void onSpawnFailure() { + // For debugging: + // ExecutorDumper.dump(this.executor, 3, ExecuteProduceConsume.class); + } + + @Override + protected Task produceTask(boolean canBlock) throws InterruptedException { + Task task; + + task = maybeProducePeerMessageTask(); + if (task != null) { + return task; + } + + final Long now = NTP.getTime(); + + task = maybeProducePeerPingTask(now); + if (task != null) { + return task; + } + + task = maybeProduceConnectPeerTask(now); + if (task != null) { + return task; + } + + task = maybeProduceBroadcastTask(now); + if (task != null) { + return task; + } + + // Only this method can block to reduce CPU spin + return maybeProduceChannelTask(canBlock); + } + + private Task maybeProducePeerMessageTask() { + for (Peer peer : getConnectedPeers()) { + Task peerTask = peer.getMessageTask(); + if (peerTask != null) { + return peerTask; + } + } + + return null; + } + + private Task maybeProducePeerPingTask(Long now) { + // Ask connected peers whether they need a ping + for (Peer peer : getHandshakedPeers()) { + Task peerTask = peer.getPingTask(now); + if (peerTask != null) { + return peerTask; + } + } + + return null; + } + + class PeerConnectTask implements ExecuteProduceConsume.Task { + private final Peer peer; + + PeerConnectTask(Peer peer) { + this.peer = peer; + } + + @Override + public void perform() throws InterruptedException { + connectPeer(peer); + } + } + + private Task maybeProduceConnectPeerTask(Long now) throws InterruptedException { + if (now == null || now < nextConnectTaskTimestamp) { + return null; + } + + if (getOutboundHandshakedPeers().size() >= minOutboundPeers) { + return null; + } + + nextConnectTaskTimestamp = now + 1000L; + + Peer targetPeer = getConnectablePeer(now); + if (targetPeer == null) { + return null; + } + + // Create connection task + return new PeerConnectTask(targetPeer); + } + + private Task maybeProduceBroadcastTask(Long now) { + if (now == null || now < nextBroadcastTimestamp) { + return null; + } + + nextBroadcastTimestamp = now + BROADCAST_INTERVAL; + return () -> Controller.getInstance().doNetworkBroadcast(); + } + + class ChannelTask implements ExecuteProduceConsume.Task { + private final SelectionKey selectionKey; + + ChannelTask(SelectionKey selectionKey) { + this.selectionKey = selectionKey; + } + + @Override + public void perform() throws InterruptedException { + try { + LOGGER.trace("Thread {} has pending channel: {}, with ops {}", + Thread.currentThread().getId(), selectionKey.channel(), selectionKey.readyOps()); + + // process pending channel task + if (selectionKey.isReadable()) { + connectionRead((SocketChannel) selectionKey.channel()); + } else if (selectionKey.isAcceptable()) { + acceptConnection((ServerSocketChannel) selectionKey.channel()); + } + + LOGGER.trace("Thread {} processed channel: {}", + Thread.currentThread().getId(), selectionKey.channel()); + } catch (CancelledKeyException e) { + LOGGER.trace("Thread {} encountered cancelled channel: {}", + Thread.currentThread().getId(), selectionKey.channel()); + } + } + + private void connectionRead(SocketChannel socketChannel) { + Peer peer = getPeerFromChannel(socketChannel); + if (peer == null) { + return; + } + + try { + peer.readChannel(); + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().toLowerCase().contains("connection reset")) { + peer.disconnect("Connection reset"); + return; + } + + LOGGER.trace("[{}] Network thread {} encountered I/O error: {}", peer.getPeerConnectionId(), + Thread.currentThread().getId(), e.getMessage(), e); + peer.disconnect("I/O error"); + } + } + } + + private Task maybeProduceChannelTask(boolean canBlock) throws InterruptedException { + final SelectionKey nextSelectionKey; + + // Synchronization here to enforce thread-safety on channelIterator + synchronized (channelSelector) { + // anything to do? + if (channelIterator == null) { + try { + if (canBlock) { + channelSelector.select(1000L); + } else { + channelSelector.selectNow(); + } + } catch (IOException e) { + LOGGER.warn("Channel selection threw IOException: {}", e.getMessage()); + return null; + } + + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException(); + } + + channelIterator = channelSelector.selectedKeys().iterator(); + } + + if (channelIterator.hasNext()) { + nextSelectionKey = channelIterator.next(); + channelIterator.remove(); + } else { + nextSelectionKey = null; + channelIterator = null; // Nothing to do so reset iterator to cause new select + } + + LOGGER.trace("Thread {}, nextSelectionKey {}, channelIterator now {}", + Thread.currentThread().getId(), nextSelectionKey, channelIterator); + } + + if (nextSelectionKey == null) { + return null; + } + + return new ChannelTask(nextSelectionKey); + } + } + + private void acceptConnection(ServerSocketChannel serverSocketChannel) throws InterruptedException { + SocketChannel socketChannel; + + try { + socketChannel = serverSocketChannel.accept(); + } catch (IOException e) { + return; + } + + // No connection actually accepted? + if (socketChannel == null) { + return; + } + PeerAddress address = PeerAddress.fromSocket(socketChannel.socket()); + List fixedNetwork = Settings.getInstance().getFixedNetwork(); + if (fixedNetwork != null && !fixedNetwork.isEmpty() && ipNotInFixedList(address, fixedNetwork)) { + try { + LOGGER.debug("Connection discarded from peer {} as not in the fixed network list", address); + socketChannel.close(); + } catch (IOException e) { + // IGNORE + } + return; + } + + final Long now = NTP.getTime(); + Peer newPeer; + + try { + if (now == null) { + LOGGER.debug("Connection discarded from peer {} due to lack of NTP sync", address); + socketChannel.close(); + return; + } + + synchronized (this.connectedPeers) { + if (connectedPeers.size() >= maxPeers) { + // We have enough peers + LOGGER.debug("Connection discarded from peer {} because the server is full", address); + socketChannel.close(); + return; + } + + LOGGER.debug("Connection accepted from peer {}", address); + + newPeer = new Peer(socketChannel, channelSelector); + this.connectedPeers.add(newPeer); + } + } catch (IOException e) { + if (socketChannel.isOpen()) { + try { + LOGGER.debug("Connection failed from peer {} while connecting/closing", address); + socketChannel.close(); + } catch (IOException ce) { + // Couldn't close? + } + } + return; + } + + this.onPeerReady(newPeer); + } + + private boolean ipNotInFixedList(PeerAddress address, List fixedNetwork) { + for (String ipAddress : fixedNetwork) { + String[] bits = ipAddress.split(":"); + if (bits.length >= 1 && bits.length <= 2 && address.getHost().equals(bits[0])) { + return false; + } + } + return true; + } + + private Peer getConnectablePeer(final Long now) throws InterruptedException { + // We can't block here so use tryRepository(). We don't NEED to connect a new peer. + try (Repository repository = RepositoryManager.tryRepository()) { + if (repository == null) { + return null; + } + + // Find an address to connect to + List peers = this.getAllKnownPeers(); + + // Don't consider peers with recent connection failures + final long lastAttemptedThreshold = now - CONNECT_FAILURE_BACKOFF; + peers.removeIf(peerData -> peerData.getLastAttempted() != null + && (peerData.getLastConnected() == null + || peerData.getLastConnected() < peerData.getLastAttempted()) + && peerData.getLastAttempted() > lastAttemptedThreshold); + + // Don't consider peers that we know loop back to ourself + synchronized (this.selfPeers) { + peers.removeIf(isSelfPeer); + } + + synchronized (this.connectedPeers) { + // Don't consider already connected peers (simple address match) + peers.removeIf(isConnectedPeer); + + // Don't consider already connected peers (resolved address match) + // XXX This might be too slow if we end up waiting a long time for hostnames to resolve via DNS + peers.removeIf(isResolvedAsConnectedPeer); + + this.checkLongestConnection(now); + } + + // Any left? + if (peers.isEmpty()) { + return null; + } + + // Pick random peer + int peerIndex = new Random().nextInt(peers.size()); + + // Pick candidate + PeerData peerData = peers.get(peerIndex); + Peer newPeer = new Peer(peerData); + + // Update connection attempt info + peerData.setLastAttempted(now); + synchronized (this.allKnownPeers) { + repository.getNetworkRepository().save(peerData); + repository.saveChanges(); + } + + return newPeer; + } catch (DataException e) { + LOGGER.error("Repository issue while finding a connectable peer", e); + return null; + } + } + + private boolean connectPeer(Peer newPeer) throws InterruptedException { + SocketChannel socketChannel = newPeer.connect(this.channelSelector); + if (socketChannel == null) { + return false; + } + + if (Thread.currentThread().isInterrupted()) { + return false; + } + + synchronized (this.connectedPeers) { + this.connectedPeers.add(newPeer); + } + + this.onPeerReady(newPeer); + + return true; + } + + private Peer getPeerFromChannel(SocketChannel socketChannel) { + synchronized (this.connectedPeers) { + for (Peer peer : this.connectedPeers) { + if (peer.getSocketChannel() == socketChannel) { + return peer; + } + } + } + + return null; + } + + private void checkLongestConnection(Long now) { + if (now == null || now < nextDisconnectionCheck) { + return; + } + + // Find peers that have reached their maximum connection age, and disconnect them + List peersToDisconnect = this.connectedPeers.stream() + .filter(peer -> !peer.isSyncInProgress()) + .filter(peer -> peer.hasReachedMaxConnectionAge()) + .collect(Collectors.toList()); + + if (peersToDisconnect != null && peersToDisconnect.size() > 0) { + for (Peer peer : peersToDisconnect) { + LOGGER.debug("Forcing disconnection of peer {} because connection age ({} ms) " + + "has reached the maximum ({} ms)", peer, peer.getConnectionAge(), peer.getMaxConnectionAge()); + peer.disconnect("Connection age too old"); + } + } + + // Check again after a minimum fixed interval + nextDisconnectionCheck = now + DISCONNECTION_CHECK_INTERVAL; + } + + // Peer callbacks + + protected void wakeupChannelSelector() { + this.channelSelector.wakeup(); + } + + protected boolean verify(byte[] signature, byte[] message) { + return Crypto.verify(this.edPublicKeyParams.getEncoded(), signature, message); + } + + protected byte[] sign(byte[] message) { + return Crypto.sign(this.edPrivateKeyParams, message); + } + + protected byte[] getSharedSecret(byte[] publicKey) { + return Crypto.getSharedSecret(this.edPrivateKeyParams.getEncoded(), publicKey); + } + + /** + * Called when Peer's thread has setup and is ready to process messages + */ + public void onPeerReady(Peer peer) { + onHandshakingMessage(peer, null, Handshake.STARTED); + } + + public void onDisconnect(Peer peer) { + // Notify Controller + Controller.getInstance().onPeerDisconnect(peer); + if (peer.getConnectionEstablishedTime() > 0L) { + LOGGER.debug("[{}] Disconnected from peer {}", peer.getPeerConnectionId(), peer); + } else { + LOGGER.debug("[{}] Failed to connect to peer {}", peer.getPeerConnectionId(), peer); + } + + synchronized (this.connectedPeers) { + this.connectedPeers.remove(peer); + } + } + + public void peerMisbehaved(Peer peer) { + PeerData peerData = peer.getPeerData(); + peerData.setLastMisbehaved(NTP.getTime()); + + // Only update repository if outbound peer + if (peer.isOutbound()) { + try (Repository repository = RepositoryManager.getRepository()) { + synchronized (this.allKnownPeers) { + repository.getNetworkRepository().save(peerData); + repository.saveChanges(); + } + } catch (DataException e) { + LOGGER.warn("Repository issue while updating peer synchronization info", e); + } + } + } + + /** + * Called when a new message arrives for a peer. message can be null if called after connection + */ + public void onMessage(Peer peer, Message message) { + if (message != null) { + LOGGER.trace("[{}} Processing {} message with ID {} from peer {}", peer.getPeerConnectionId(), + message.getType().name(), message.getId(), peer); + } + + Handshake handshakeStatus = peer.getHandshakeStatus(); + if (handshakeStatus != Handshake.COMPLETED) { + onHandshakingMessage(peer, message, handshakeStatus); + return; + } + + // Should be non-handshaking messages from now on + + // Ordered by message type value + switch (message.getType()) { + case GET_PEERS: + onGetPeersMessage(peer, message); + break; + + case PING: + onPingMessage(peer, message); + break; + + case HELLO: + case CHALLENGE: + case RESPONSE: + LOGGER.debug("[{}] Unexpected handshaking message {} from peer {}", peer.getPeerConnectionId(), + message.getType().name(), peer); + peer.disconnect("unexpected handshaking message"); + return; + + case PEERS_V2: + onPeersV2Message(peer, message); + break; + + default: + // Bump up to controller for possible action + Controller.getInstance().onNetworkMessage(peer, message); + break; + } + } + + private void onHandshakingMessage(Peer peer, Message message, Handshake handshakeStatus) { + try { + // Still handshaking + LOGGER.trace("[{}] Handshake status {}, message {} from peer {}", peer.getPeerConnectionId(), + handshakeStatus.name(), (message != null ? message.getType().name() : "null"), peer); + + // Check message type is as expected + if (handshakeStatus.expectedMessageType != null + && message.getType() != handshakeStatus.expectedMessageType) { + LOGGER.debug("[{}] Unexpected {} message from {}, expected {}", peer.getPeerConnectionId(), + message.getType().name(), peer, handshakeStatus.expectedMessageType); + peer.disconnect("unexpected message"); + return; + } + + Handshake newHandshakeStatus = handshakeStatus.onMessage(peer, message); + + if (newHandshakeStatus == null) { + // Handshake failure + LOGGER.debug("[{}] Handshake failure with peer {} message {}", peer.getPeerConnectionId(), peer, + message.getType().name()); + peer.disconnect("handshake failure"); + return; + } + + if (peer.isOutbound()) { + // If we made outbound connection then we need to act first + newHandshakeStatus.action(peer); + } else { + // We have inbound connection so we need to respond in kind with what we just received + handshakeStatus.action(peer); + } + peer.setHandshakeStatus(newHandshakeStatus); + + if (newHandshakeStatus == Handshake.COMPLETED) { + this.onHandshakeCompleted(peer); + } + } finally { + peer.resetHandshakeMessagePending(); + } + } + + private void onGetPeersMessage(Peer peer, Message message) { + // Send our known peers + if (!peer.sendMessage(this.buildPeersMessage(peer))) { + peer.disconnect("failed to send peers list"); + } + } + + private void onPingMessage(Peer peer, Message message) { + PingMessage pingMessage = (PingMessage) message; + + // Generate 'pong' using same ID + PingMessage pongMessage = new PingMessage(); + pongMessage.setId(pingMessage.getId()); + + if (!peer.sendMessage(pongMessage)) { + peer.disconnect("failed to send ping reply"); + } + } + + private void onPeersV2Message(Peer peer, Message message) { + PeersV2Message peersV2Message = (PeersV2Message) message; + + List peerV2Addresses = peersV2Message.getPeerAddresses(); + + // First entry contains remote peer's listen port but empty address. + int peerPort = peerV2Addresses.get(0).getPort(); + peerV2Addresses.remove(0); + + // If inbound peer, use listen port and socket address to recreate first entry + if (!peer.isOutbound()) { + String host = peer.getPeerData().getAddress().getHost(); + PeerAddress sendingPeerAddress = PeerAddress.fromString(host + ":" + peerPort); + LOGGER.trace("PEERS_V2 sending peer's listen address: {}", sendingPeerAddress.toString()); + peerV2Addresses.add(0, sendingPeerAddress); + } + + opportunisticMergePeers(peer.toString(), peerV2Addresses); + } + + protected void onHandshakeCompleted(Peer peer) { + LOGGER.debug("[{}] Handshake completed with peer {} on {}", peer.getPeerConnectionId(), peer, + peer.getPeersVersionString()); + + // Are we already connected to this peer? + Peer existingPeer = getHandshakedPeerWithPublicKey(peer.getPeersPublicKey()); + // NOTE: actual object reference compare, not Peer.equals() + if (existingPeer != peer) { + LOGGER.info("[{}] We already have a connection with peer {} - discarding", + peer.getPeerConnectionId(), peer); + peer.disconnect("existing connection"); + return; + } + + // Make a note that we've successfully completed handshake (and when) + peer.getPeerData().setLastConnected(NTP.getTime()); + + // Update connection info for outbound peers only + if (peer.isOutbound()) { + try (Repository repository = RepositoryManager.getRepository()) { + synchronized (this.allKnownPeers) { + repository.getNetworkRepository().save(peer.getPeerData()); + repository.saveChanges(); + } + } catch (DataException e) { + LOGGER.error("[{}] Repository issue while trying to update outbound peer {}", + peer.getPeerConnectionId(), peer, e); + } + } + + // Process any pending signature requests, as this peer may have been connected for this purpose only + List pendingSignatureRequests = new ArrayList<>(peer.getPendingSignatureRequests()); + if (pendingSignatureRequests != null && !pendingSignatureRequests.isEmpty()) { + for (byte[] signature : pendingSignatureRequests) { + this.requestDataFromConnectedPeer(peer, signature); + peer.removePendingSignatureRequest(signature); + } + } + + // FUTURE: we may want to disconnect from this peer if we've finished requesting data from it + + // Start regular pings + peer.startPings(); + + // Only the outbound side needs to send anything (after we've received handshake-completing response). + // (If inbound sent anything here, it's possible it could be processed out-of-order with handshake message). + + if (peer.isOutbound()) { + // Send our height + Message heightMessage = buildHeightMessage(peer, Controller.getInstance().getChainTip()); + if (!peer.sendMessage(heightMessage)) { + peer.disconnect("failed to send height/info"); + return; + } + + // Send our peers list + Message peersMessage = this.buildPeersMessage(peer); + if (!peer.sendMessage(peersMessage)) { + peer.disconnect("failed to send peers list"); + } + + // Request their peers list + Message getPeersMessage = new GetPeersMessage(); + if (!peer.sendMessage(getPeersMessage)) { + peer.disconnect("failed to request peers list"); + } + } + + // Ask Controller if they want to do anything + Controller.getInstance().onPeerHandshakeCompleted(peer); + } + + // Message-building calls + + /** + * Returns PEERS message made from peers we've connected to recently, and this node's details + */ + public Message buildPeersMessage(Peer peer) { + List knownPeers = this.getAllKnownPeers(); + + // Filter out peers that we've not connected to ever or within X milliseconds + final long connectionThreshold = NTP.getTime() - RECENT_CONNECTION_THRESHOLD; + Predicate notRecentlyConnected = peerData -> { + final Long lastAttempted = peerData.getLastAttempted(); + final Long lastConnected = peerData.getLastConnected(); + + if (lastAttempted == null || lastConnected == null) { + return true; + } + + if (lastConnected < lastAttempted) { + return true; + } + + if (lastConnected < connectionThreshold) { + return true; + } + + return false; + }; + knownPeers.removeIf(notRecentlyConnected); + + List peerAddresses = new ArrayList<>(); + + for (PeerData peerData : knownPeers) { + try { + InetAddress address = InetAddress.getByName(peerData.getAddress().getHost()); + + // Don't send 'local' addresses if peer is not 'local'. + // e.g. don't send localhost:9084 to node4.qortal.org + if (!peer.isLocal() && Peer.isAddressLocal(address)) { + continue; + } + + peerAddresses.add(peerData.getAddress()); + } catch (UnknownHostException e) { + // Couldn't resolve hostname to IP address so discard + } + } + + // New format PEERS_V2 message that supports hostnames, IPv6 and ports + return new PeersV2Message(peerAddresses); + } + + public Message buildHeightMessage(Peer peer, BlockData blockData) { + // HEIGHT_V2 contains way more useful info + return new HeightV2Message(blockData.getHeight(), blockData.getSignature(), + blockData.getTimestamp(), blockData.getMinterPublicKey()); + } + + public Message buildNewTransactionMessage(Peer peer, TransactionData transactionData) { + // In V2 we send out transaction signature only and peers can decide whether to request the full transaction + return new TransactionSignaturesMessage(Collections.singletonList(transactionData.getSignature())); + } + + public Message buildGetUnconfirmedTransactionsMessage(Peer peer) { + return new GetUnconfirmedTransactionsMessage(); + } + + + // External IP / peerAddress tracking + + public void ourPeerAddressUpdated(String peerAddress) { + if (peerAddress == null || peerAddress.isEmpty()) { + return; + } + + // Validate IP address + String[] parts = peerAddress.split(":"); + if (parts.length != 2) { + return; + } + String host = parts[0]; + try { + InetAddress addr = InetAddress.getByName(host); + if (addr.isAnyLocalAddress() || addr.isSiteLocalAddress()) { + // Ignore local addresses + return; + } + } catch (UnknownHostException e) { + return; + } + + // Add to the list + this.ourExternalIpAddressHistory.add(host); + + // Limit to 25 entries + while (this.ourExternalIpAddressHistory.size() > 25) { + this.ourExternalIpAddressHistory.remove(0); + } + + // Now take a copy of the IP address history so it can be safely iterated + // Without this, another thread could remove an element, resulting in an exception + List ipAddressHistory = new ArrayList<>(this.ourExternalIpAddressHistory); + + // If we've had 10 consecutive matching addresses, and they're different from + // our stored IP address value, treat it as updated. + int consecutiveReadingsRequired = 10; + int size = ipAddressHistory.size(); + if (size < consecutiveReadingsRequired) { + // Need at least 10 readings + return; + } + + // Count the number of consecutive IP address readings + String lastReading = null; + int consecutiveReadings = 0; + for (int i = size-1; i >= 0; i--) { + String reading = ipAddressHistory.get(i); + if (lastReading != null) { + if (Objects.equals(reading, lastReading)) { + consecutiveReadings++; + } + else { + consecutiveReadings = 0; + } + } + lastReading = reading; + } + + if (consecutiveReadings >= consecutiveReadingsRequired) { + // Last 10 readings were the same - i.e. more than one peer agreed on the new IP address... + String ip = ipAddressHistory.get(size - 1); + if (ip != null && !Objects.equals(ip, "null")) { + if (!Objects.equals(ip, this.ourExternalIpAddress)) { + // ... and the readings were different to our current recorded value, so + // update our external IP address value + this.ourExternalIpAddress = ip; + this.onExternalIpUpdate(ip); + } + } + } + } + + public void onExternalIpUpdate(String ipAddress) { + LOGGER.info("External IP address updated to {}", ipAddress); + + //ArbitraryDataManager.getInstance().broadcastHostedSignatureList(); + } + + + // Peer-management calls + + public void noteToSelf(Peer peer) { + LOGGER.info("[{}] No longer considering peer address {} as it connects to self", + peer.getPeerConnectionId(), peer); + + synchronized (this.selfPeers) { + this.selfPeers.add(peer.getPeerData().getAddress()); + } + } + + public boolean forgetPeer(PeerAddress peerAddress) throws DataException { + int numDeleted; + + synchronized (this.allKnownPeers) { + this.allKnownPeers.removeIf(peerData -> peerData.getAddress().equals(peerAddress)); + + try (Repository repository = RepositoryManager.getRepository()) { + numDeleted = repository.getNetworkRepository().delete(peerAddress); + repository.saveChanges(); + } + } + + disconnectPeer(peerAddress); + + return numDeleted != 0; + } + + public int forgetAllPeers() throws DataException { + int numDeleted; + + synchronized (this.allKnownPeers) { + this.allKnownPeers.clear(); + + try (Repository repository = RepositoryManager.getRepository()) { + numDeleted = repository.getNetworkRepository().deleteAllPeers(); + repository.saveChanges(); + } + } + + for (Peer peer : this.getConnectedPeers()) { + peer.disconnect("to be forgotten"); + } + + return numDeleted; + } + + private void disconnectPeer(PeerAddress peerAddress) { + // Disconnect peer + try { + InetSocketAddress knownAddress = peerAddress.toSocketAddress(); + + List peers = this.getConnectedPeers(); + peers.removeIf(peer -> !Peer.addressEquals(knownAddress, peer.getResolvedAddress())); + + for (Peer peer : peers) { + peer.disconnect("to be forgotten"); + } + } catch (UnknownHostException e) { + // Unknown host isn't going to match any of our connected peers so ignore + } + } + + // Network-wide calls + + public void prunePeers() throws DataException { + final Long now = NTP.getTime(); + if (now == null) { + return; + } + + // Disconnect peers that are stuck during handshake + List handshakePeers = this.getConnectedPeers(); + + // Disregard peers that have completed handshake or only connected recently + handshakePeers.removeIf(peer -> peer.getHandshakeStatus() == Handshake.COMPLETED + || peer.getConnectionTimestamp() == null || peer.getConnectionTimestamp() > now - HANDSHAKE_TIMEOUT); + + for (Peer peer : handshakePeers) { + peer.disconnect(String.format("handshake timeout at %s", peer.getHandshakeStatus().name())); + } + + // Prune 'old' peers from repository... + // Pruning peers isn't critical so no need to block for a repository instance. + try (Repository repository = RepositoryManager.tryRepository()) { + if (repository == null) { + return; + } + + synchronized (this.allKnownPeers) { + // Fetch all known peers + List peers = new ArrayList<>(this.allKnownPeers); + + // 'Old' peers: + // We attempted to connect within the last day + // but we last managed to connect over a week ago. + Predicate isNotOldPeer = peerData -> { + if (peerData.getLastAttempted() == null + || peerData.getLastAttempted() < now - OLD_PEER_ATTEMPTED_PERIOD) { + return true; + } + + if (peerData.getLastConnected() == null + || peerData.getLastConnected() > now - OLD_PEER_CONNECTION_PERIOD) { + return true; + } + + return false; + }; + + // Disregard peers that are NOT 'old' + peers.removeIf(isNotOldPeer); + + // Don't consider already connected peers (simple address match) + synchronized (this.connectedPeers) { + peers.removeIf(isConnectedPeer); + } + + for (PeerData peerData : peers) { + LOGGER.debug("Deleting old peer {} from repository", peerData.getAddress().toString()); + repository.getNetworkRepository().delete(peerData.getAddress()); + + // Delete from known peer cache too + this.allKnownPeers.remove(peerData); + } + + repository.saveChanges(); + } + } + } + + public boolean mergePeers(String addedBy, long addedWhen, List peerAddresses) throws DataException { + mergePeersLock.lock(); + + try (Repository repository = RepositoryManager.getRepository()) { + return this.mergePeers(repository, addedBy, addedWhen, peerAddresses); + } finally { + mergePeersLock.unlock(); + } + } + + private void opportunisticMergePeers(String addedBy, List peerAddresses) { + final Long addedWhen = NTP.getTime(); + if (addedWhen == null) { + return; + } + + // Serialize using lock to prevent repository deadlocks + if (!mergePeersLock.tryLock()) { + return; + } + + try { + // Merging peers isn't critical so don't block for a repository instance. + try (Repository repository = RepositoryManager.tryRepository()) { + if (repository == null) { + return; + } + + this.mergePeers(repository, addedBy, addedWhen, peerAddresses); + + } catch (DataException e) { + // Already logged by this.mergePeers() + } + } finally { + mergePeersLock.unlock(); + } + } + + private boolean mergePeers(Repository repository, String addedBy, long addedWhen, List peerAddresses) + throws DataException { + List fixedNetwork = Settings.getInstance().getFixedNetwork(); + if (fixedNetwork != null && !fixedNetwork.isEmpty()) { + return false; + } + List newPeers; + synchronized (this.allKnownPeers) { + for (PeerData knownPeerData : this.allKnownPeers) { + // Filter out duplicates, without resolving via DNS + Predicate isKnownAddress = peerAddress -> knownPeerData.getAddress().equals(peerAddress); + peerAddresses.removeIf(isKnownAddress); + } + + if (peerAddresses.isEmpty()) { + return false; + } + + // Add leftover peer addresses to known peers list + newPeers = peerAddresses.stream() + .map(peerAddress -> new PeerData(peerAddress, addedWhen, addedBy)) + .collect(Collectors.toList()); + + this.allKnownPeers.addAll(newPeers); + + try { + // Save new peers into database + for (PeerData peerData : newPeers) { + LOGGER.info("Adding new peer {} to repository", peerData.getAddress()); + repository.getNetworkRepository().save(peerData); + } + + repository.saveChanges(); + } catch (DataException e) { + LOGGER.error("Repository issue while merging peers list from {}", addedBy, e); + throw e; + } + + return true; + } + } + + public void broadcast(Function peerMessageBuilder) { + class Broadcaster implements Runnable { + private final Random random = new Random(); + + private List targetPeers; + private Function peerMessageBuilder; + + Broadcaster(List targetPeers, Function peerMessageBuilder) { + this.targetPeers = targetPeers; + this.peerMessageBuilder = peerMessageBuilder; + } + + @Override + public void run() { + Thread.currentThread().setName("Network Broadcast"); + + for (Peer peer : targetPeers) { + // Very short sleep to reduce strain, improve multi-threading and catch interrupts + try { + Thread.sleep(random.nextInt(20) + 20L); + } catch (InterruptedException e) { + break; + } + + Message message = peerMessageBuilder.apply(peer); + + if (message == null) { + continue; + } + + if (!peer.sendMessage(message)) { + peer.disconnect("failed to broadcast message"); + } + } + + Thread.currentThread().setName("Network Broadcast (dormant)"); + } + } + + try { + broadcastExecutor.execute(new Broadcaster(this.getHandshakedPeers(), peerMessageBuilder)); + } catch (RejectedExecutionException e) { + // Can't execute - probably because we're shutting down, so ignore + } + } + + // Shutdown + + public void shutdown() { + // Close listen socket to prevent more incoming connections + if (this.serverChannel.isOpen()) { + try { + this.serverChannel.close(); + } catch (IOException e) { + // Not important + } + } + + // Stop processing threads + try { + if (!this.networkEPC.shutdown(5000)) { + LOGGER.warn("Network threads failed to terminate"); + } + } catch (InterruptedException e) { + LOGGER.warn("Interrupted while waiting for networking threads to terminate"); + } + + // Stop broadcasts + this.broadcastExecutor.shutdownNow(); + try { + if (!this.broadcastExecutor.awaitTermination(1000, TimeUnit.MILLISECONDS)) { + LOGGER.warn("Broadcast threads failed to terminate"); + } + } catch (InterruptedException e) { + LOGGER.warn("Interrupted while waiting for broadcast threads failed to terminate"); + } + + // Close all peer connections + for (Peer peer : this.getConnectedPeers()) { + peer.shutdown(); + } + } } diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index 968b9e51c..da4a70a92 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -1,595 +1,883 @@ package org.qortal.network; -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.SocketTimeoutException; -import java.net.StandardSocketOptions; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; -import java.nio.channels.SelectionKey; -import java.nio.channels.Selector; -import java.nio.channels.SocketChannel; -import java.security.SecureRandom; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - +import com.google.common.hash.HashCode; +import com.google.common.net.HostAndPort; +import com.google.common.net.InetAddresses; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.data.block.CommonBlockData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; import org.qortal.network.message.ChallengeMessage; import org.qortal.network.message.Message; -import org.qortal.network.message.PingMessage; import org.qortal.network.message.Message.MessageException; import org.qortal.network.message.Message.MessageType; +import org.qortal.network.message.PingMessage; import org.qortal.settings.Settings; import org.qortal.utils.ExecuteProduceConsume; import org.qortal.utils.NTP; -import com.google.common.net.HostAndPort; -import com.google.common.net.InetAddresses; +import java.io.IOException; +import java.net.*; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; +import java.security.SecureRandom; +import java.util.*; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; // For managing one peer public class Peer { + private static final Logger LOGGER = LogManager.getLogger(Peer.class); - private static final Logger LOGGER = LogManager.getLogger(Peer.class); - - /** Maximum time to allow connect() to remote peer to complete. (ms) */ - private static final int CONNECT_TIMEOUT = 2000; // ms - - /** Maximum time to wait for a message reply to arrive from peer. (ms) */ - private static final int RESPONSE_TIMEOUT = 2000; // ms - - /** - * Interval between PING messages to a peer. (ms) - *

- * Just under every 30s is usually ideal to keep NAT mappings refreshed. - */ - private static final int PING_INTERVAL = 20_000; // ms - - private volatile boolean isStopping = false; - - private SocketChannel socketChannel = null; - private InetSocketAddress resolvedAddress = null; - /** True if remote address is loopback/link-local/site-local, false otherwise. */ - private boolean isLocal; - - private final Object byteBufferLock = new Object(); - private ByteBuffer byteBuffer; - - private Map> replyQueues; - private LinkedBlockingQueue pendingMessages; - - /** True if we created connection to peer, false if we accepted incoming connection from peer. */ - private final boolean isOutbound; - - private final Object handshakingLock = new Object(); - private Handshake handshakeStatus = Handshake.STARTED; - private volatile boolean handshakeMessagePending = false; - - /** Timestamp of when socket was accepted, or connected. */ - private Long connectionTimestamp = null; - - /** Last PING message round-trip time (ms). */ - private Long lastPing = null; - /** When last PING message was sent, or null if pings not started yet. */ - private Long lastPingSent; - - byte[] ourChallenge; - - // Peer info - - private final Object peerInfoLock = new Object(); - - private String peersNodeId; - private byte[] peersPublicKey; - private byte[] peersChallenge; - - private PeerData peerData = null; - - /** Peer's value of connectionTimestamp. */ - private Long peersConnectionTimestamp = null; - - /** Version string as reported by peer. */ - private String peersVersionString = null; - /** Numeric version of peer. */ - private Long peersVersion = null; - - /** Latest block info as reported by peer. */ - private PeerChainTipData peersChainTipData; - - // Constructors - - /** Construct unconnected, outbound Peer using socket address in peer data */ - public Peer(PeerData peerData) { - this.isOutbound = true; - this.peerData = peerData; - } - - /** Construct Peer using existing, connected socket */ - public Peer(SocketChannel socketChannel, Selector channelSelector) throws IOException { - this.isOutbound = false; - this.socketChannel = socketChannel; - sharedSetup(channelSelector); - - this.resolvedAddress = ((InetSocketAddress) socketChannel.socket().getRemoteSocketAddress()); - this.isLocal = isAddressLocal(this.resolvedAddress.getAddress()); - - PeerAddress peerAddress = PeerAddress.fromSocket(socketChannel.socket()); - this.peerData = new PeerData(peerAddress); - } - - // Getters / setters - - public boolean isStopping() { - return this.isStopping; - } - - public SocketChannel getSocketChannel() { - return this.socketChannel; - } - - public InetSocketAddress getResolvedAddress() { - return this.resolvedAddress; - } - - public boolean isLocal() { - return this.isLocal; - } - - public boolean isOutbound() { - return this.isOutbound; - } - - public Handshake getHandshakeStatus() { - synchronized (this.handshakingLock) { - return this.handshakeStatus; - } - } - - /*package*/ void setHandshakeStatus(Handshake handshakeStatus) { - synchronized (this.handshakingLock) { - this.handshakeStatus = handshakeStatus; - } - } - - /*package*/ void resetHandshakeMessagePending() { - this.handshakeMessagePending = false; - } - - public PeerData getPeerData() { - synchronized (this.peerInfoLock) { - return this.peerData; - } - } - - public Long getConnectionTimestamp() { - synchronized (this.peerInfoLock) { - return this.connectionTimestamp; - } - } - - public String getPeersVersionString() { - synchronized (this.peerInfoLock) { - return this.peersVersionString; - } - } - - public Long getPeersVersion() { - synchronized (this.peerInfoLock) { - return this.peersVersion; - } - } - - /*package*/ void setPeersVersion(String versionString, long version) { - synchronized (this.peerInfoLock) { - this.peersVersionString = versionString; - this.peersVersion = version; - } - } - - public Long getPeersConnectionTimestamp() { - synchronized (this.peerInfoLock) { - return this.peersConnectionTimestamp; - } - } - - /*package*/ void setPeersConnectionTimestamp(Long peersConnectionTimestamp) { - synchronized (this.peerInfoLock) { - this.peersConnectionTimestamp = peersConnectionTimestamp; - } - } - - public Long getLastPing() { - synchronized (this.peerInfoLock) { - return this.lastPing; - } - } - - /*package*/ void setLastPing(long lastPing) { - synchronized (this.peerInfoLock) { - this.lastPing = lastPing; - } - } - - /*package*/ byte[] getOurChallenge() { - return this.ourChallenge; - } - - public String getPeersNodeId() { - synchronized (this.peerInfoLock) { - return this.peersNodeId; - } - } - - /*package*/ void setPeersNodeId(String peersNodeId) { - synchronized (this.peerInfoLock) { - this.peersNodeId = peersNodeId; - } - } - - public byte[] getPeersPublicKey() { - synchronized (this.peerInfoLock) { - return this.peersPublicKey; - } - } - - /*package*/ void setPeersPublicKey(byte[] peerPublicKey) { - synchronized (this.peerInfoLock) { - this.peersPublicKey = peerPublicKey; - } - } - - public byte[] getPeersChallenge() { - synchronized (this.peerInfoLock) { - return this.peersChallenge; - } - } - - /*package*/ void setPeersChallenge(byte[] peersChallenge) { - synchronized (this.peerInfoLock) { - this.peersChallenge = peersChallenge; - } - } - - public PeerChainTipData getChainTipData() { - synchronized (this.peerInfoLock) { - return this.peersChainTipData; - } - } - - public void setChainTipData(PeerChainTipData chainTipData) { - synchronized (this.peerInfoLock) { - this.peersChainTipData = chainTipData; - } - } - - /*package*/ void queueMessage(Message message) { - if (!this.pendingMessages.offer(message)) - LOGGER.info(() -> String.format("No room to queue message from peer %s - discarding", this)); - } - - @Override - public String toString() { - // Easier, and nicer output, than peer.getRemoteSocketAddress() - return this.peerData.getAddress().toString(); - } - - // Processing - - private void sharedSetup(Selector channelSelector) throws IOException { - this.connectionTimestamp = NTP.getTime(); - this.socketChannel.setOption(StandardSocketOptions.TCP_NODELAY, true); - this.socketChannel.configureBlocking(false); - this.socketChannel.register(channelSelector, SelectionKey.OP_READ); - this.byteBuffer = null; // Defer allocation to when we need it, to save memory. Sorry GC! - this.replyQueues = Collections.synchronizedMap(new HashMap>()); - this.pendingMessages = new LinkedBlockingQueue<>(); - - Random random = new SecureRandom(); - this.ourChallenge = new byte[ChallengeMessage.CHALLENGE_LENGTH]; - random.nextBytes(this.ourChallenge); - } - - public SocketChannel connect(Selector channelSelector) { - LOGGER.trace(() -> String.format("Connecting to peer %s", this)); - - try { - this.resolvedAddress = this.peerData.getAddress().toSocketAddress(); - this.isLocal = isAddressLocal(this.resolvedAddress.getAddress()); - - this.socketChannel = SocketChannel.open(); - this.socketChannel.socket().connect(resolvedAddress, CONNECT_TIMEOUT); - } catch (SocketTimeoutException e) { - LOGGER.trace(String.format("Connection timed out to peer %s", this)); - return null; - } catch (UnknownHostException e) { - LOGGER.trace(String.format("Connection failed to unresolved peer %s", this)); - return null; - } catch (IOException e) { - LOGGER.trace(String.format("Connection failed to peer %s", this)); - return null; - } - - try { - LOGGER.debug(() -> String.format("Connected to peer %s", this)); - sharedSetup(channelSelector); - return socketChannel; - } catch (IOException e) { - LOGGER.trace(String.format("Post-connection setup failed, peer %s", this)); - try { - socketChannel.close(); - } catch (IOException ce) { - // Failed to close? - } - return null; - } - } - - /** - * Attempt to buffer bytes from socketChannel. - * - * @throws IOException - */ - /* package */ void readChannel() throws IOException { - synchronized (this.byteBufferLock) { - while(true) { - if (!this.socketChannel.isOpen() || this.socketChannel.socket().isClosed()) - return; - - // Do we need to allocate byteBuffer? - if (this.byteBuffer == null) - this.byteBuffer = ByteBuffer.allocate(Network.getInstance().getMaxMessageSize()); - - final int bytesRead = this.socketChannel.read(this.byteBuffer); - if (bytesRead == -1) { - this.disconnect("EOF"); - return; - } - - LOGGER.trace(() -> String.format("Received %d bytes from peer %s", bytesRead, this)); - final boolean wasByteBufferFull = !this.byteBuffer.hasRemaining(); - - while (true) { - final Message message; - - // Can we build a message from buffer now? - try { - message = Message.fromByteBuffer(this.byteBuffer); - } catch (MessageException e) { - LOGGER.debug(String.format("%s, from peer %s", e.getMessage(), this)); - this.disconnect(e.getMessage()); - return; - } - - if (message == null && bytesRead == 0 && !wasByteBufferFull) { - // No complete message in buffer, no more bytes to read from socket even though there was room to read bytes - - // If byteBuffer is empty then we can deallocate it, to save memory, albeit costing GC - if (this.byteBuffer.remaining() == this.byteBuffer.capacity()) - this.byteBuffer = null; - - return; - } - - if (message == null) - // No complete message in buffer, but maybe more bytes to read from socket - break; - - LOGGER.trace(() -> String.format("Received %s message with ID %d from peer %s", message.getType().name(), message.getId(), this)); - - BlockingQueue queue = this.replyQueues.get(message.getId()); - if (queue != null) { - // Adding message to queue will unblock thread waiting for response - this.replyQueues.get(message.getId()).add(message); - // Consumed elsewhere - continue; - } - - // No thread waiting for message so we need to pass it up to network layer - - // Add message to pending queue - if (!this.pendingMessages.offer(message)) { - LOGGER.info(String.format("No room to queue message from peer %s - discarding", this)); - return; - } - - // Prematurely end any blocking channel select so that new messages can be processed. - // This might cause this.socketChannel.read() above to return zero into bytesRead. - Network.getInstance().wakeupChannelSelector(); - } - } - } - } - - /* package */ ExecuteProduceConsume.Task getMessageTask() { - /* - * If we are still handshaking and there is a message yet to be processed then - * don't produce another message task. This allows us to process handshake - * messages sequentially. - */ - if (this.handshakeMessagePending) - return null; - - final Message nextMessage = this.pendingMessages.poll(); - - if (nextMessage == null) - return null; - - LOGGER.trace(() -> String.format("Produced %s message task from peer %s", nextMessage.getType().name(), this)); - - if (this.handshakeStatus != Handshake.COMPLETED) - this.handshakeMessagePending = true; - - // Return a task to process message in queue - return () -> Network.getInstance().onMessage(this, nextMessage); - } - - /** - * Attempt to send Message to peer. - * - * @param message - * @return true if message successfully sent; false otherwise - */ - public boolean sendMessage(Message message) { - if (!this.socketChannel.isOpen()) - return false; - - try { - // Send message - LOGGER.trace(() -> String.format("Sending %s message with ID %d to peer %s", message.getType().name(), message.getId(), this)); - - ByteBuffer outputBuffer = ByteBuffer.wrap(message.toBytes()); - - synchronized (this.socketChannel) { - while (outputBuffer.hasRemaining()) { - int bytesWritten = this.socketChannel.write(outputBuffer); - - if (bytesWritten == 0) - // Underlying socket's internal buffer probably full, - // so wait a short while for bytes to actually be transmitted over the wire - this.socketChannel.wait(1L); - } - } - } catch (MessageException e) { - LOGGER.warn(String.format("Failed to send %s message with ID %d to peer %s: %s", message.getType().name(), message.getId(), this, e.getMessage())); - } catch (IOException e) { - // Send failure - return false; - } catch (InterruptedException e) { - // Likely shutdown scenario - so exit - return false; - } - - // Sent OK - return true; - } - - /** - * Send message to peer and await response. - *

- * Message is assigned a random ID and sent. If a response with matching ID is received then it is returned to caller. - *

- * If no response with matching ID within timeout, or some other error/exception occurs, then return null.
- * (Assume peer will be rapidly disconnected after this). - * - * @param message - * @return Message if valid response received; null if not or error/exception occurs - * @throws InterruptedException - */ - public Message getResponse(Message message) throws InterruptedException { - BlockingQueue blockingQueue = new ArrayBlockingQueue<>(1); - - // Assign random ID to this message - Random random = new Random(); - int id; - do { - id = random.nextInt(Integer.MAX_VALUE - 1) + 1; - - // Put queue into map (keyed by message ID) so we can poll for a response - // If putIfAbsent() doesn't return null, then this ID is already taken - } while (this.replyQueues.putIfAbsent(id, blockingQueue) != null); - message.setId(id); - - // Try to send message - if (!this.sendMessage(message)) { - this.replyQueues.remove(id); - return null; - } - - try { - return blockingQueue.poll(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - } finally { - this.replyQueues.remove(id); - } - } - - /* package */ void startPings() { - // Replacing initial null value allows getPingTask() to start sending pings. - LOGGER.trace(() -> String.format("Enabling pings for peer %s", this)); - this.lastPingSent = NTP.getTime(); - } - - /* package */ ExecuteProduceConsume.Task getPingTask(Long now) { - // Pings not enabled yet? - if (now == null || this.lastPingSent == null) - return null; - - // Time to send another ping? - if (now < this.lastPingSent + PING_INTERVAL) - return null; // Not yet - - // Not strictly true, but prevents this peer from being immediately chosen again - this.lastPingSent = now; - - return () -> { - PingMessage pingMessage = new PingMessage(); - Message message = this.getResponse(pingMessage); - - if (message == null || message.getType() != MessageType.PING) { - LOGGER.debug(() -> String.format("Didn't receive reply from %s for PING ID %d", this, pingMessage.getId())); - this.disconnect("no ping received"); - return; - } - - this.setLastPing(NTP.getTime() - now); - }; - } - - public void disconnect(String reason) { - if (!isStopping) - LOGGER.debug(() -> String.format("Disconnecting peer %s: %s", this, reason)); - - this.shutdown(); - - Network.getInstance().onDisconnect(this); - } - - public void shutdown() { - if (!isStopping) - LOGGER.debug(() -> String.format("Shutting down peer %s", this)); - - isStopping = true; - - if (this.socketChannel.isOpen()) { - try { - this.socketChannel.shutdownOutput(); - this.socketChannel.close(); - } catch (IOException e) { - LOGGER.debug(String.format("IOException while trying to close peer %s", this)); - } - } - } - - // Utility methods - - /** Returns true if ports and addresses (or hostnames) match */ - public static boolean addressEquals(InetSocketAddress knownAddress, InetSocketAddress peerAddress) { - if (knownAddress.getPort() != peerAddress.getPort()) - return false; - - return knownAddress.getHostString().equalsIgnoreCase(peerAddress.getHostString()); - } - - public static InetSocketAddress parsePeerAddress(String peerAddress) throws IllegalArgumentException { - HostAndPort hostAndPort = HostAndPort.fromString(peerAddress).requireBracketsForIPv6(); - - // HostAndPort doesn't try to validate host so we do extra checking here - InetAddress address = InetAddresses.forString(hostAndPort.getHost()); - - return new InetSocketAddress(address, hostAndPort.getPortOrDefault(Settings.getInstance().getDefaultListenPort())); - } - - /** Returns true if address is loopback/link-local/site-local, false otherwise. */ - public static boolean isAddressLocal(InetAddress address) { - return address.isLoopbackAddress() || address.isLinkLocalAddress() || address.isSiteLocalAddress(); - } + /** + * Maximum time to allow connect() to remote peer to complete. (ms) + */ + private static final int CONNECT_TIMEOUT = 2000; // ms + /** + * Maximum time to wait for a message reply to arrive from peer. (ms) + */ + private static final int RESPONSE_TIMEOUT = 3000; // ms + + /** + * Maximum time to wait for a peer to respond with blocks (ms) + */ + public static final int FETCH_BLOCKS_TIMEOUT = 10000; + + /** + * Interval between PING messages to a peer. (ms) + *

+ * Just under every 30s is usually ideal to keep NAT mappings refreshed. + */ + private static final int PING_INTERVAL = 20_000; // ms + + private volatile boolean isStopping = false; + + private SocketChannel socketChannel = null; + private InetSocketAddress resolvedAddress = null; + /** + * True if remote address is loopback/link-local/site-local, false otherwise. + */ + private boolean isLocal; + + private final UUID peerConnectionId = UUID.randomUUID(); + private final Object byteBufferLock = new Object(); + private ByteBuffer byteBuffer; + + private Map> replyQueues; + private LinkedBlockingQueue pendingMessages; + + /** + * True if we created connection to peer, false if we accepted incoming connection from peer. + */ + private final boolean isOutbound; + + private final Object handshakingLock = new Object(); + private Handshake handshakeStatus = Handshake.STARTED; + private volatile boolean handshakeMessagePending = false; + private long handshakeComplete = -1L; + private long maxConnectionAge = 0L; + + /** + * Timestamp of when socket was accepted, or connected. + */ + private Long connectionTimestamp = null; + + /** + * Last PING message round-trip time (ms). + */ + private Long lastPing = null; + /** + * When last PING message was sent, or null if pings not started yet. + */ + private Long lastPingSent; + + byte[] ourChallenge; + + private boolean syncInProgress = false; + + + /* Pending signature requests */ + private List pendingSignatureRequests = Collections.synchronizedList(new ArrayList<>()); + + + // Versioning + public static final Pattern VERSION_PATTERN = Pattern.compile(Controller.VERSION_PREFIX + + "(\\d{1,3})\\.(\\d{1,5})\\.(\\d{1,5})"); + + // Peer info + + private final Object peerInfoLock = new Object(); + + private String peersNodeId; + private byte[] peersPublicKey; + private byte[] peersChallenge; + + private PeerData peerData = null; + + /** + * Peer's value of connectionTimestamp. + */ + private Long peersConnectionTimestamp = null; + + /** + * Version string as reported by peer. + */ + private String peersVersionString = null; + /** + * Numeric version of peer. + */ + private Long peersVersion = null; + + /** + * Latest block info as reported by peer. + */ + private PeerChainTipData peersChainTipData; + + /** + * Our common block with this peer + */ + private CommonBlockData commonBlockData; + + // Constructors + + /** + * Construct unconnected, outbound Peer using socket address in peer data + */ + public Peer(PeerData peerData) { + this.isOutbound = true; + this.peerData = peerData; + } + + /** + * Construct Peer using existing, connected socket + */ + public Peer(SocketChannel socketChannel, Selector channelSelector) throws IOException { + this.isOutbound = false; + this.socketChannel = socketChannel; + sharedSetup(channelSelector); + + this.resolvedAddress = ((InetSocketAddress) socketChannel.socket().getRemoteSocketAddress()); + this.isLocal = isAddressLocal(this.resolvedAddress.getAddress()); + + PeerAddress peerAddress = PeerAddress.fromSocket(socketChannel.socket()); + this.peerData = new PeerData(peerAddress); + } + + // Getters / setters + + public boolean isStopping() { + return this.isStopping; + } + + public SocketChannel getSocketChannel() { + return this.socketChannel; + } + + public InetSocketAddress getResolvedAddress() { + return this.resolvedAddress; + } + + public boolean isLocal() { + return this.isLocal; + } + + public boolean isOutbound() { + return this.isOutbound; + } + + public Handshake getHandshakeStatus() { + synchronized (this.handshakingLock) { + return this.handshakeStatus; + } + } + + protected void setHandshakeStatus(Handshake handshakeStatus) { + synchronized (this.handshakingLock) { + this.handshakeStatus = handshakeStatus; + if (handshakeStatus.equals(Handshake.COMPLETED)) { + this.handshakeComplete = System.currentTimeMillis(); + this.generateRandomMaxConnectionAge(); + } + } + } + + private void generateRandomMaxConnectionAge() { + // Retrieve the min and max connection time from the settings, and calculate the range + final int minPeerConnectionTime = Settings.getInstance().getMinPeerConnectionTime(); + final int maxPeerConnectionTime = Settings.getInstance().getMaxPeerConnectionTime(); + final int peerConnectionTimeRange = maxPeerConnectionTime - minPeerConnectionTime; + + // Generate a random number between the min and the max + Random random = new Random(); + this.maxConnectionAge = (random.nextInt(peerConnectionTimeRange) + minPeerConnectionTime) * 1000L; + LOGGER.debug(String.format("[%s] Generated max connection age for peer %s. Min: %ds, max: %ds, range: %ds, random max: %dms", this.peerConnectionId, this, minPeerConnectionTime, maxPeerConnectionTime, peerConnectionTimeRange, this.maxConnectionAge)); + + } + + protected void resetHandshakeMessagePending() { + this.handshakeMessagePending = false; + } + + public PeerData getPeerData() { + synchronized (this.peerInfoLock) { + return this.peerData; + } + } + + public Long getConnectionTimestamp() { + synchronized (this.peerInfoLock) { + return this.connectionTimestamp; + } + } + + public String getPeersVersionString() { + synchronized (this.peerInfoLock) { + return this.peersVersionString; + } + } + + public Long getPeersVersion() { + synchronized (this.peerInfoLock) { + return this.peersVersion; + } + } + + protected void setPeersVersion(String versionString, long version) { + synchronized (this.peerInfoLock) { + this.peersVersionString = versionString; + this.peersVersion = version; + } + } + + public Long getPeersConnectionTimestamp() { + synchronized (this.peerInfoLock) { + return this.peersConnectionTimestamp; + } + } + + protected void setPeersConnectionTimestamp(Long peersConnectionTimestamp) { + synchronized (this.peerInfoLock) { + this.peersConnectionTimestamp = peersConnectionTimestamp; + } + } + + public Long getLastPing() { + synchronized (this.peerInfoLock) { + return this.lastPing; + } + } + + protected void setLastPing(long lastPing) { + synchronized (this.peerInfoLock) { + this.lastPing = lastPing; + } + } + + protected byte[] getOurChallenge() { + return this.ourChallenge; + } + + public String getPeersNodeId() { + synchronized (this.peerInfoLock) { + return this.peersNodeId; + } + } + + protected void setPeersNodeId(String peersNodeId) { + synchronized (this.peerInfoLock) { + this.peersNodeId = peersNodeId; + } + } + + public byte[] getPeersPublicKey() { + synchronized (this.peerInfoLock) { + return this.peersPublicKey; + } + } + + protected void setPeersPublicKey(byte[] peerPublicKey) { + synchronized (this.peerInfoLock) { + this.peersPublicKey = peerPublicKey; + } + } + + public byte[] getPeersChallenge() { + synchronized (this.peerInfoLock) { + return this.peersChallenge; + } + } + + protected void setPeersChallenge(byte[] peersChallenge) { + synchronized (this.peerInfoLock) { + this.peersChallenge = peersChallenge; + } + } + + public PeerChainTipData getChainTipData() { + synchronized (this.peerInfoLock) { + return this.peersChainTipData; + } + } + + public void setChainTipData(PeerChainTipData chainTipData) { + synchronized (this.peerInfoLock) { + this.peersChainTipData = chainTipData; + } + } + + public CommonBlockData getCommonBlockData() { + synchronized (this.peerInfoLock) { + return this.commonBlockData; + } + } + + public void setCommonBlockData(CommonBlockData commonBlockData) { + synchronized (this.peerInfoLock) { + this.commonBlockData = commonBlockData; + } + } + + protected void queueMessage(Message message) { + if (!this.pendingMessages.offer(message)) { + LOGGER.info("[{}] No room to queue message from peer {} - discarding", this.peerConnectionId, this); + } + } + + public boolean isSyncInProgress() { + return this.syncInProgress; + } + + public void setSyncInProgress(boolean syncInProgress) { + this.syncInProgress = syncInProgress; + } + + + // Pending signature requests + + public void addPendingSignatureRequest(byte[] signature) { + // Check if we already have this signature in the list + for (byte[] existingSignature : this.pendingSignatureRequests) { + if (Arrays.equals(existingSignature, signature )) { + return; + } + } + this.pendingSignatureRequests.add(signature); + } + + public void removePendingSignatureRequest(byte[] signature) { + Iterator iterator = this.pendingSignatureRequests.iterator(); + while (iterator.hasNext()) { + byte[] existingSignature = (byte[]) iterator.next(); + if (Arrays.equals(existingSignature, signature)) { + iterator.remove(); + } + } + } + + public List getPendingSignatureRequests() { + return this.pendingSignatureRequests; + } + + + @Override + public String toString() { + // Easier, and nicer output, than peer.getRemoteSocketAddress() + return this.peerData.getAddress().toString(); + } + + // Processing + + private void sharedSetup(Selector channelSelector) throws IOException { + this.connectionTimestamp = NTP.getTime(); + this.socketChannel.setOption(StandardSocketOptions.TCP_NODELAY, true); + this.socketChannel.configureBlocking(false); + this.socketChannel.register(channelSelector, SelectionKey.OP_READ); + this.byteBuffer = null; // Defer allocation to when we need it, to save memory. Sorry GC! + this.replyQueues = Collections.synchronizedMap(new HashMap>()); + this.pendingMessages = new LinkedBlockingQueue<>(); + + Random random = new SecureRandom(); + this.ourChallenge = new byte[ChallengeMessage.CHALLENGE_LENGTH]; + random.nextBytes(this.ourChallenge); + } + + public SocketChannel connect(Selector channelSelector) { + LOGGER.trace("[{}] Connecting to peer {}", this.peerConnectionId, this); + + try { + this.resolvedAddress = this.peerData.getAddress().toSocketAddress(); + this.isLocal = isAddressLocal(this.resolvedAddress.getAddress()); + + this.socketChannel = SocketChannel.open(); + this.socketChannel.socket().connect(resolvedAddress, CONNECT_TIMEOUT); + } catch (SocketTimeoutException e) { + LOGGER.trace("[{}] Connection timed out to peer {}", this.peerConnectionId, this); + return null; + } catch (UnknownHostException e) { + LOGGER.trace("[{}] Connection failed to unresolved peer {}", this.peerConnectionId, this); + return null; + } catch (IOException e) { + LOGGER.trace("[{}] Connection failed to peer {}", this.peerConnectionId, this); + return null; + } + + try { + LOGGER.debug("[{}] Connected to peer {}", this.peerConnectionId, this); + sharedSetup(channelSelector); + return socketChannel; + } catch (IOException e) { + LOGGER.trace("[{}] Post-connection setup failed, peer {}", this.peerConnectionId, this); + try { + socketChannel.close(); + } catch (IOException ce) { + // Failed to close? + } + return null; + } + } + + /** + * Attempt to buffer bytes from socketChannel. + * + * @throws IOException If this channel is not yet connected + */ + protected void readChannel() throws IOException { + synchronized (this.byteBufferLock) { + while (true) { + if (!this.socketChannel.isOpen() || this.socketChannel.socket().isClosed()) { + return; + } + + // Do we need to allocate byteBuffer? + if (this.byteBuffer == null) { + this.byteBuffer = ByteBuffer.allocate(Network.getInstance().getMaxMessageSize()); + } + + final int priorPosition = this.byteBuffer.position(); + final int bytesRead = this.socketChannel.read(this.byteBuffer); + if (bytesRead == -1) { + if (priorPosition > 0) { + this.disconnect("EOF - read " + priorPosition + " bytes"); + } else { + this.disconnect("EOF - failed to read any data"); + } + return; + } + + if (LOGGER.isTraceEnabled()) { + if (bytesRead > 0) { + byte[] leadingBytes = new byte[Math.min(bytesRead, 8)]; + this.byteBuffer.asReadOnlyBuffer().position(priorPosition).get(leadingBytes); + String leadingHex = HashCode.fromBytes(leadingBytes).toString(); + + LOGGER.trace("[{}] Received {} bytes, starting {}, into byteBuffer[{}] from peer {}", + this.peerConnectionId, bytesRead, leadingHex, priorPosition, this); + } else { + LOGGER.trace("[{}] Received {} bytes into byteBuffer[{}] from peer {}", this.peerConnectionId, + bytesRead, priorPosition, this); + } + } + final boolean wasByteBufferFull = !this.byteBuffer.hasRemaining(); + + while (true) { + final Message message; + + // Can we build a message from buffer now? + ByteBuffer readOnlyBuffer = this.byteBuffer.asReadOnlyBuffer().flip(); + try { + message = Message.fromByteBuffer(readOnlyBuffer); + } catch (MessageException e) { + LOGGER.debug("[{}] {}, from peer {}", this.peerConnectionId, e.getMessage(), this); + this.disconnect(e.getMessage()); + return; + } + + if (message == null && bytesRead == 0 && !wasByteBufferFull) { + // No complete message in buffer, no more bytes to read from socket + // even though there was room to read bytes + + /* DISABLED + // If byteBuffer is empty then we can deallocate it, to save memory, albeit costing GC + if (this.byteBuffer.remaining() == this.byteBuffer.capacity()) { + this.byteBuffer = null; + } + */ + + return; + } + + if (message == null) { + // No complete message in buffer, but maybe more bytes to read from socket + break; + } + + LOGGER.trace("[{}] Received {} message with ID {} from peer {}", this.peerConnectionId, + message.getType().name(), message.getId(), this); + + // Tidy up buffers: + this.byteBuffer.flip(); + // Read-only, flipped buffer's position will be after end of message, so copy that + this.byteBuffer.position(readOnlyBuffer.position()); + // Copy bytes after read message to front of buffer, + // adjusting position accordingly, reset limit to capacity + this.byteBuffer.compact(); + + BlockingQueue queue = this.replyQueues.get(message.getId()); + if (queue != null) { + // Adding message to queue will unblock thread waiting for response + this.replyQueues.get(message.getId()).add(message); + // Consumed elsewhere + continue; + } + + // No thread waiting for message so we need to pass it up to network layer + + // Add message to pending queue + if (!this.pendingMessages.offer(message)) { + LOGGER.info("[{}] No room to queue message from peer {} - discarding", + this.peerConnectionId, this); + return; + } + + // Prematurely end any blocking channel select so that new messages can be processed. + // This might cause this.socketChannel.read() above to return zero into bytesRead. + Network.getInstance().wakeupChannelSelector(); + } + } + } + } + + protected ExecuteProduceConsume.Task getMessageTask() { + /* + * If we are still handshaking and there is a message yet to be processed then + * don't produce another message task. This allows us to process handshake + * messages sequentially. + */ + if (this.handshakeMessagePending) { + return null; + } + + final Message nextMessage = this.pendingMessages.poll(); + + if (nextMessage == null) { + return null; + } + + LOGGER.trace("[{}] Produced {} message task from peer {}", this.peerConnectionId, + nextMessage.getType().name(), this); + + if (this.handshakeStatus != Handshake.COMPLETED) { + this.handshakeMessagePending = true; + } + + // Return a task to process message in queue + return () -> Network.getInstance().onMessage(this, nextMessage); + } + + /** + * Attempt to send Message to peer, using default RESPONSE_TIMEOUT. + * + * @param message message to be sent + * @return true if message successfully sent; false otherwise + */ + public boolean sendMessage(Message message) { + return this.sendMessageWithTimeout(message, RESPONSE_TIMEOUT); + } + + /** + * Attempt to send Message to peer, using custom timeout. + * + * @param message message to be sent + * @return true if message successfully sent; false otherwise + */ + public boolean sendMessageWithTimeout(Message message, int timeout) { + if (!this.socketChannel.isOpen()) { + return false; + } + + try { + // Send message + LOGGER.trace("[{}] Sending {} message with ID {} to peer {}", this.peerConnectionId, + message.getType().name(), message.getId(), this); + + ByteBuffer outputBuffer = ByteBuffer.wrap(message.toBytes()); + + synchronized (this.socketChannel) { + final long sendStart = System.currentTimeMillis(); + long totalBytes = 0; + + while (outputBuffer.hasRemaining()) { + int bytesWritten = this.socketChannel.write(outputBuffer); + totalBytes += bytesWritten; + + LOGGER.trace("[{}] Sent {} bytes of {} message with ID {} to peer {} ({} total)", this.peerConnectionId, + bytesWritten, message.getType().name(), message.getId(), this, totalBytes); + + if (bytesWritten == 0) { + // Underlying socket's internal buffer probably full, + // so wait a short while for bytes to actually be transmitted over the wire + + /* + * NOSONAR squid:S2276 - we don't want to use this.socketChannel.wait() + * as this releases the lock held by synchronized() above + * and would allow another thread to send another message, + * potentially interleaving them on-the-wire, causing checksum failures + * and connection loss. + */ + Thread.sleep(1L); //NOSONAR squid:S2276 + + if (System.currentTimeMillis() - sendStart > timeout) { + // We've taken too long to send this message + return false; + } + } + } + } + } catch (MessageException e) { + LOGGER.warn("[{}] Failed to send {} message with ID {} to peer {}: {}", this.peerConnectionId, + message.getType().name(), message.getId(), this, e.getMessage()); + return false; + } catch (IOException | InterruptedException e) { + // Send failure + return false; + } + + // Sent OK + return true; + } + + /** + * Send message to peer and await response, using default RESPONSE_TIMEOUT. + *

+ * Message is assigned a random ID and sent. + * If a response with matching ID is received then it is returned to caller. + *

+ * If no response with matching ID within timeout, or some other error/exception occurs, + * then return null.
+ * (Assume peer will be rapidly disconnected after this). + * + * @param message message to send + * @return Message if valid response received; null if not or error/exception occurs + * @throws InterruptedException if interrupted while waiting + */ + public Message getResponse(Message message) throws InterruptedException { + return getResponseWithTimeout(message, RESPONSE_TIMEOUT); + } + + /** + * Send message to peer and await response. + *

+ * Message is assigned a random ID and sent. + * If a response with matching ID is received then it is returned to caller. + *

+ * If no response with matching ID within timeout, or some other error/exception occurs, + * then return null.
+ * (Assume peer will be rapidly disconnected after this). + * + * @param message message to send + * @return Message if valid response received; null if not or error/exception occurs + * @throws InterruptedException if interrupted while waiting + */ + public Message getResponseWithTimeout(Message message, int timeout) throws InterruptedException { + BlockingQueue blockingQueue = new ArrayBlockingQueue<>(1); + + // Assign random ID to this message + Random random = new Random(); + int id; + do { + id = random.nextInt(Integer.MAX_VALUE - 1) + 1; + + // Put queue into map (keyed by message ID) so we can poll for a response + // If putIfAbsent() doesn't return null, then this ID is already taken + } while (this.replyQueues.putIfAbsent(id, blockingQueue) != null); + message.setId(id); + + // Try to send message + if (!this.sendMessageWithTimeout(message, timeout)) { + this.replyQueues.remove(id); + return null; + } + + try { + return blockingQueue.poll(timeout, TimeUnit.MILLISECONDS); + } finally { + this.replyQueues.remove(id); + } + } + + protected void startPings() { + // Replacing initial null value allows getPingTask() to start sending pings. + LOGGER.trace("[{}] Enabling pings for peer {}", this.peerConnectionId, this); + this.lastPingSent = NTP.getTime(); + } + + protected ExecuteProduceConsume.Task getPingTask(Long now) { + // Pings not enabled yet? + if (now == null || this.lastPingSent == null) { + return null; + } + + // Time to send another ping? + if (now < this.lastPingSent + PING_INTERVAL) { + return null; // Not yet + } + + // Not strictly true, but prevents this peer from being immediately chosen again + this.lastPingSent = now; + + return () -> { + PingMessage pingMessage = new PingMessage(); + Message message = this.getResponse(pingMessage); + + if (message == null || message.getType() != MessageType.PING) { + LOGGER.debug("[{}] Didn't receive reply from {} for PING ID {}", this.peerConnectionId, this, + pingMessage.getId()); + this.disconnect("no ping received"); + return; + } + + this.setLastPing(NTP.getTime() - now); + }; + } + + public void disconnect(String reason) { + if (!isStopping) { + LOGGER.debug("[{}] Disconnecting peer {} after {}: {}", this.peerConnectionId, this, + getConnectionAge(), reason); + } + this.shutdown(); + + Network.getInstance().onDisconnect(this); + } + + public void shutdown() { + if (!isStopping) { + LOGGER.debug("[{}] Shutting down peer {}", this.peerConnectionId, this); + } + isStopping = true; + + if (this.socketChannel.isOpen()) { + try { + this.socketChannel.shutdownOutput(); + this.socketChannel.close(); + } catch (IOException e) { + LOGGER.debug("[{}] IOException while trying to close peer {}", this.peerConnectionId, this); + } + } + } + + + // Minimum version + + public boolean isAtLeastVersion(String minVersionString) { + if (minVersionString == null) { + return false; + } + + // Add the version prefix + minVersionString = Controller.VERSION_PREFIX + minVersionString; + + Matcher matcher = VERSION_PATTERN.matcher(minVersionString); + if (!matcher.lookingAt()) { + return false; + } + + // We're expecting 3 positive shorts, so we can convert 1.2.3 into 0x0100020003 + long minVersion = 0; + for (int g = 1; g <= 3; ++g) { + long value = Long.parseLong(matcher.group(g)); + + if (value < 0 || value > Short.MAX_VALUE) { + return false; + } + + minVersion <<= 16; + minVersion |= value; + } + + return this.getPeersVersion() >= minVersion; + } + + + // Common block data + + public boolean canUseCachedCommonBlockData() { + PeerChainTipData peerChainTipData = this.getChainTipData(); + CommonBlockData commonBlockData = this.getCommonBlockData(); + + if (peerChainTipData != null && commonBlockData != null) { + PeerChainTipData commonBlockChainTipData = commonBlockData.getChainTipData(); + if (peerChainTipData.getLastBlockSignature() != null && commonBlockChainTipData != null + && commonBlockChainTipData.getLastBlockSignature() != null) { + if (Arrays.equals(peerChainTipData.getLastBlockSignature(), + commonBlockChainTipData.getLastBlockSignature())) { + return true; + } + } + } + return false; + } + + + // Utility methods + + /** + * Returns true if ports and addresses (or hostnames) match + */ + public static boolean addressEquals(InetSocketAddress knownAddress, InetSocketAddress peerAddress) { + if (knownAddress.getPort() != peerAddress.getPort()) { + return false; + } + + return knownAddress.getHostString().equalsIgnoreCase(peerAddress.getHostString()); + } + + public static InetSocketAddress parsePeerAddress(String peerAddress) throws IllegalArgumentException { + HostAndPort hostAndPort = HostAndPort.fromString(peerAddress).requireBracketsForIPv6(); + + // HostAndPort doesn't try to validate host so we do extra checking here + InetAddress address = InetAddresses.forString(hostAndPort.getHost()); + + int defaultPort = Settings.getInstance().getDefaultListenPort(); + return new InetSocketAddress(address, hostAndPort.getPortOrDefault(defaultPort)); + } + + /** + * Returns true if address is loopback/link-local/site-local, false otherwise. + */ + public static boolean isAddressLocal(InetAddress address) { + return address.isLoopbackAddress() || address.isLinkLocalAddress() || address.isSiteLocalAddress(); + } + + public UUID getPeerConnectionId() { + return peerConnectionId; + } + + public long getConnectionEstablishedTime() { + return handshakeComplete; + } + + public long getConnectionAge() { + if (handshakeComplete > 0L) { + return System.currentTimeMillis() - handshakeComplete; + } + return handshakeComplete; + } + + public long getMaxConnectionAge() { + return maxConnectionAge; + } + + public boolean hasReachedMaxConnectionAge() { + return this.getConnectionAge() > this.getMaxConnectionAge(); + } } diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java new file mode 100644 index 000000000..008b3eddb --- /dev/null +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java @@ -0,0 +1,90 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import org.qortal.transform.TransformationException; +import org.qortal.transform.Transformer; +import org.qortal.utils.Serialization; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class ArbitraryDataFileListMessage extends Message { + + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + private static final int HASH_LENGTH = Transformer.SHA256_LENGTH; + + private final byte[] signature; + private final List hashes; + + public ArbitraryDataFileListMessage(byte[] signature, List hashes) { + super(MessageType.ARBITRARY_DATA_FILE_LIST); + + this.signature = signature; + this.hashes = hashes; + } + + public ArbitraryDataFileListMessage(int id, byte[] signature, List hashes) { + super(id, MessageType.ARBITRARY_DATA_FILE_LIST); + + this.signature = signature; + this.hashes = hashes; + } + + public List getHashes() { + return this.hashes; + } + + public byte[] getSignature() { + return this.signature; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException, TransformationException { + byte[] signature = new byte[SIGNATURE_LENGTH]; + bytes.get(signature); + + int count = bytes.getInt(); + + if (bytes.remaining() != count * HASH_LENGTH) + return null; + + List hashes = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + + byte[] hash = new byte[HASH_LENGTH]; + bytes.get(hash); + hashes.add(hash); + } + + return new ArbitraryDataFileListMessage(id, signature, hashes); + } + + @Override + protected byte[] toData() { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(this.signature); + + bytes.write(Ints.toByteArray(this.hashes.size())); + + for (byte[] hash : this.hashes) { + bytes.write(hash); + } + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } + } + + public ArbitraryDataFileListMessage cloneWithNewId(int newId) { + ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes); + clone.setId(newId); + return clone; + } + +} diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java new file mode 100644 index 000000000..b9f24e29e --- /dev/null +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java @@ -0,0 +1,96 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.repository.DataException; +import org.qortal.transform.Transformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; + +public class ArbitraryDataFileMessage extends Message { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileMessage.class); + + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + + private final byte[] signature; + private final ArbitraryDataFile arbitraryDataFile; + + public ArbitraryDataFileMessage(byte[] signature, ArbitraryDataFile arbitraryDataFile) { + super(MessageType.ARBITRARY_DATA_FILE); + + this.signature = signature; + this.arbitraryDataFile = arbitraryDataFile; + } + + public ArbitraryDataFileMessage(int id, byte[] signature, ArbitraryDataFile arbitraryDataFile) { + super(id, MessageType.ARBITRARY_DATA_FILE); + + this.signature = signature; + this.arbitraryDataFile = arbitraryDataFile; + } + + public ArbitraryDataFile getArbitraryDataFile() { + return this.arbitraryDataFile; + } + + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { + byte[] signature = new byte[SIGNATURE_LENGTH]; + byteBuffer.get(signature); + + int dataLength = byteBuffer.getInt(); + + if (byteBuffer.remaining() != dataLength) + return null; + + byte[] data = new byte[dataLength]; + byteBuffer.get(data); + + try { + ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data, signature); + return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile); + } + catch (DataException e) { + LOGGER.info("Unable to process received file: {}", e.getMessage()); + return null; + } + } + + @Override + protected byte[] toData() { + if (this.arbitraryDataFile == null) { + return null; + } + + byte[] data = this.arbitraryDataFile.getBytes(); + if (data == null) { + return null; + } + + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(signature); + + bytes.write(Ints.toByteArray(data.length)); + + bytes.write(data); + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } + } + + public ArbitraryDataFileMessage cloneWithNewId(int newId) { + ArbitraryDataFileMessage clone = new ArbitraryDataFileMessage(this.signature, this.arbitraryDataFile); + clone.setId(newId); + return clone; + } + +} diff --git a/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java b/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java new file mode 100644 index 000000000..1f980b3cb --- /dev/null +++ b/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java @@ -0,0 +1,92 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import org.qortal.data.network.PeerData; +import org.qortal.transform.TransformationException; +import org.qortal.transform.Transformer; +import org.qortal.utils.Serialization; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class ArbitrarySignaturesMessage extends Message { + + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + + private String peerAddress; + private int requestHops; + private List signatures; + + public ArbitrarySignaturesMessage(String peerAddress, int requestHops, List signatures) { + this(-1, peerAddress, requestHops, signatures); + } + + private ArbitrarySignaturesMessage(int id, String peerAddress, int requestHops, List signatures) { + super(id, MessageType.ARBITRARY_SIGNATURES); + + this.peerAddress = peerAddress; + this.requestHops = requestHops; + this.signatures = signatures; + } + + public String getPeerAddress() { + return this.peerAddress; + } + + public List getSignatures() { + return this.signatures; + } + + public int getRequestHops() { + return this.requestHops; + } + + public void setRequestHops(int requestHops) { + this.requestHops = requestHops; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException, TransformationException { + String peerAddress = Serialization.deserializeSizedStringV2(bytes, PeerData.MAX_PEER_ADDRESS_SIZE); + + int requestHops = bytes.getInt(); + + int signatureCount = bytes.getInt(); + + if (bytes.remaining() != signatureCount * SIGNATURE_LENGTH) + return null; + + List signatures = new ArrayList<>(); + for (int i = 0; i < signatureCount; ++i) { + byte[] signature = new byte[SIGNATURE_LENGTH]; + bytes.get(signature); + signatures.add(signature); + } + + return new ArbitrarySignaturesMessage(id, peerAddress, requestHops, signatures); + } + + @Override + protected byte[] toData() { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + Serialization.serializeSizedStringV2(bytes, this.peerAddress); + + bytes.write(Ints.toByteArray(this.requestHops)); + + bytes.write(Ints.toByteArray(this.signatures.size())); + + for (byte[] signature : this.signatures) + bytes.write(signature); + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/network/message/BlockMessage.java b/src/main/java/org/qortal/network/message/BlockMessage.java index 8ca86ee68..b07dc8b12 100644 --- a/src/main/java/org/qortal/network/message/BlockMessage.java +++ b/src/main/java/org/qortal/network/message/BlockMessage.java @@ -34,6 +34,7 @@ public BlockMessage(Block block) { super(MessageType.BLOCK); this.block = block; + this.blockData = block.getBlockData(); this.height = block.getBlockData().getHeight(); } @@ -93,4 +94,10 @@ protected byte[] toData() { } } + public BlockMessage cloneWithNewId(int newId) { + BlockMessage clone = new BlockMessage(this.block); + clone.setId(newId); + return clone; + } + } diff --git a/src/main/java/org/qortal/network/message/CachedBlockMessage.java b/src/main/java/org/qortal/network/message/CachedBlockMessage.java new file mode 100644 index 000000000..e5029ab01 --- /dev/null +++ b/src/main/java/org/qortal/network/message/CachedBlockMessage.java @@ -0,0 +1,70 @@ +package org.qortal.network.message; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; + +import org.qortal.block.Block; +import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformer; + +import com.google.common.primitives.Ints; + +// This is an OUTGOING-only Message which more readily lends itself to being cached +public class CachedBlockMessage extends Message { + + private Block block = null; + private byte[] cachedBytes = null; + + public CachedBlockMessage(Block block) { + super(MessageType.BLOCK); + + this.block = block; + } + + public CachedBlockMessage(byte[] cachedBytes) { + super(MessageType.BLOCK); + + this.block = null; + this.cachedBytes = cachedBytes; + } + + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { + throw new UnsupportedOperationException("CachedBlockMessage is for outgoing messages only"); + } + + @Override + protected byte[] toData() { + // Already serialized? + if (this.cachedBytes != null) + return cachedBytes; + + if (this.block == null) + return null; + + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(Ints.toByteArray(this.block.getBlockData().getHeight())); + + bytes.write(BlockTransformer.toBytes(this.block)); + + this.cachedBytes = bytes.toByteArray(); + // We no longer need source Block + // and Block contains repository handle which is highly likely to be invalid after this call + this.block = null; + + return this.cachedBytes; + } catch (TransformationException | IOException e) { + return null; + } + } + + public CachedBlockMessage cloneWithNewId(int newId) { + CachedBlockMessage clone = new CachedBlockMessage(this.cachedBytes); + clone.setId(newId); + return clone; + } + +} diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java new file mode 100644 index 000000000..af19eec16 --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java @@ -0,0 +1,113 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.qortal.transform.Transformer; +import org.qortal.transform.transaction.TransactionTransformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import static org.qortal.transform.Transformer.INT_LENGTH; +import static org.qortal.transform.Transformer.LONG_LENGTH; + +public class GetArbitraryDataFileListMessage extends Message { + + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; + + private final byte[] signature; + private List hashes; + private final long requestTime; + private int requestHops; + + public GetArbitraryDataFileListMessage(byte[] signature, List hashes, long requestTime, int requestHops) { + this(-1, signature, hashes, requestTime, requestHops); + } + + private GetArbitraryDataFileListMessage(int id, byte[] signature, List hashes, long requestTime, int requestHops) { + super(id, MessageType.GET_ARBITRARY_DATA_FILE_LIST); + + this.signature = signature; + this.hashes = hashes; + this.requestTime = requestTime; + this.requestHops = requestHops; + } + + public byte[] getSignature() { + return this.signature; + } + + public List getHashes() { + return this.hashes; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + byte[] signature = new byte[SIGNATURE_LENGTH]; + + bytes.get(signature); + + long requestTime = bytes.getLong(); + + int requestHops = bytes.getInt(); + + List hashes = null; + if (bytes.hasRemaining()) { + int hashCount = bytes.getInt(); + + if (bytes.remaining() != hashCount * HASH_LENGTH) { + return null; + } + + hashes = new ArrayList<>(); + for (int i = 0; i < hashCount; ++i) { + byte[] hash = new byte[HASH_LENGTH]; + bytes.get(hash); + hashes.add(hash); + } + } + + return new GetArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops); + } + + @Override + protected byte[] toData() { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(this.signature); + + bytes.write(Longs.toByteArray(this.requestTime)); + + bytes.write(Ints.toByteArray(this.requestHops)); + + if (this.hashes != null) { + bytes.write(Ints.toByteArray(this.hashes.size())); + + for (byte[] hash : this.hashes) { + bytes.write(hash); + } + } + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } + } + + public long getRequestTime() { + return this.requestTime; + } + + public int getRequestHops() { + return this.requestHops; + } + public void setRequestHops(int requestHops) { + this.requestHops = requestHops; + } + +} diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java new file mode 100644 index 000000000..809b983d4 --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java @@ -0,0 +1,66 @@ +package org.qortal.network.message; + +import org.qortal.transform.Transformer; +import org.qortal.transform.transaction.TransactionTransformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; + +public class GetArbitraryDataFileMessage extends Message { + + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; + + private final byte[] signature; + private final byte[] hash; + + public GetArbitraryDataFileMessage(byte[] signature, byte[] hash) { + this(-1, signature, hash); + } + + private GetArbitraryDataFileMessage(int id, byte[] signature, byte[] hash) { + super(id, MessageType.GET_ARBITRARY_DATA_FILE); + + this.signature = signature; + this.hash = hash; + } + + public byte[] getSignature() { + return this.signature; + } + + public byte[] getHash() { + return this.hash; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + if (bytes.remaining() != HASH_LENGTH + SIGNATURE_LENGTH) + return null; + + byte[] signature = new byte[SIGNATURE_LENGTH]; + bytes.get(signature); + + byte[] hash = new byte[HASH_LENGTH]; + bytes.get(hash); + + return new GetArbitraryDataFileMessage(id, signature, hash); + } + + @Override + protected byte[] toData() { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(this.signature); + + bytes.write(this.hash); + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java index dcb24fec4..23c21bc50 100644 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java @@ -6,6 +6,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import org.qortal.data.network.OnlineAccountData; import org.qortal.transform.Transformer; @@ -14,7 +15,7 @@ import com.google.common.primitives.Longs; public class GetOnlineAccountsMessage extends Message { - private static final int MAX_ACCOUNT_COUNT = 1000; + private static final int MAX_ACCOUNT_COUNT = 5000; private List onlineAccounts; @@ -25,7 +26,7 @@ public GetOnlineAccountsMessage(List onlineAccounts) { private GetOnlineAccountsMessage(int id, List onlineAccounts) { super(id, MessageType.GET_ONLINE_ACCOUNTS); - this.onlineAccounts = onlineAccounts; + this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList()); } public List getOnlineAccounts() { @@ -35,12 +36,9 @@ public List getOnlineAccounts() { public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { final int accountCount = bytes.getInt(); - if (accountCount > MAX_ACCOUNT_COUNT) - return null; - List onlineAccounts = new ArrayList<>(accountCount); - for (int i = 0; i < accountCount; ++i) { + for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) { long timestamp = bytes.getLong(); byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java new file mode 100644 index 000000000..709f97820 --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java @@ -0,0 +1,117 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.qortal.data.network.OnlineAccountData; +import org.qortal.transform.Transformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * For requesting online accounts info from remote peer, given our list of online accounts. + * + * Different format to V1: + * V1 is: number of entries, then timestamp + pubkey for each entry + * V2 is: groups of: number of entries, timestamp, then pubkey for each entry + * + * Also V2 only builds online accounts message once! + */ +public class GetOnlineAccountsV2Message extends Message { + private List onlineAccounts; + private byte[] cachedData; + + public GetOnlineAccountsV2Message(List onlineAccounts) { + this(-1, onlineAccounts); + } + + private GetOnlineAccountsV2Message(int id, List onlineAccounts) { + super(id, MessageType.GET_ONLINE_ACCOUNTS_V2); + + this.onlineAccounts = onlineAccounts; + } + + public List getOnlineAccounts() { + return this.onlineAccounts; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + int accountCount = bytes.getInt(); + + List onlineAccounts = new ArrayList<>(accountCount); + + while (accountCount > 0) { + long timestamp = bytes.getLong(); + + for (int i = 0; i < accountCount; ++i) { + byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + bytes.get(publicKey); + + onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey)); + } + + if (bytes.hasRemaining()) { + accountCount = bytes.getInt(); + } else { + // we've finished + accountCount = 0; + } + } + + return new GetOnlineAccountsV2Message(id, onlineAccounts); + } + + @Override + protected synchronized byte[] toData() { + if (this.cachedData != null) + return this.cachedData; + + // Shortcut in case we have no online accounts + if (this.onlineAccounts.isEmpty()) { + this.cachedData = Ints.toByteArray(0); + return this.cachedData; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (int i = 0; i < this.onlineAccounts.size(); ++i) { + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + Long timestamp = onlineAccountData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + this.onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH; + + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (int i = 0; i < this.onlineAccounts.size(); ++i) { + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + + if (onlineAccountData.getTimestamp() == timestamp) + bytes.write(onlineAccountData.getPublicKey()); + } + } + + this.cachedData = bytes.toByteArray(); + return this.cachedData; + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/network/message/HelloMessage.java b/src/main/java/org/qortal/network/message/HelloMessage.java index 537daf48e..1b6de17db 100644 --- a/src/main/java/org/qortal/network/message/HelloMessage.java +++ b/src/main/java/org/qortal/network/message/HelloMessage.java @@ -13,16 +13,18 @@ public class HelloMessage extends Message { private final long timestamp; private final String versionString; + private final String senderPeerAddress; - private HelloMessage(int id, long timestamp, String versionString) { + private HelloMessage(int id, long timestamp, String versionString, String senderPeerAddress) { super(id, MessageType.HELLO); this.timestamp = timestamp; this.versionString = versionString; + this.senderPeerAddress = senderPeerAddress; } - public HelloMessage(long timestamp, String versionString) { - this(-1, timestamp, versionString); + public HelloMessage(long timestamp, String versionString, String senderPeerAddress) { + this(-1, timestamp, versionString, senderPeerAddress); } public long getTimestamp() { @@ -33,12 +35,22 @@ public String getVersionString() { return this.versionString; } + public String getSenderPeerAddress() { + return this.senderPeerAddress; + } + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws TransformationException { long timestamp = byteBuffer.getLong(); String versionString = Serialization.deserializeSizedString(byteBuffer, 255); - return new HelloMessage(id, timestamp, versionString); + // Sender peer address added in v3.0, so is an optional field. Older versions won't send it. + String senderPeerAddress = null; + if (byteBuffer.hasRemaining()) { + senderPeerAddress = Serialization.deserializeSizedString(byteBuffer, 255); + } + + return new HelloMessage(id, timestamp, versionString, senderPeerAddress); } @Override @@ -49,6 +61,8 @@ protected byte[] toData() throws IOException { Serialization.serializeSizedString(bytes, this.versionString); + Serialization.serializeSizedString(bytes, this.senderPeerAddress); + return bytes.toByteArray(); } diff --git a/src/main/java/org/qortal/network/message/Message.java b/src/main/java/org/qortal/network/message/Message.java index 9dfdc6bca..6c89a0ddf 100644 --- a/src/main/java/org/qortal/network/message/Message.java +++ b/src/main/java/org/qortal/network/message/Message.java @@ -25,7 +25,7 @@ public abstract class Message { private static final int MAGIC_LENGTH = 4; private static final int CHECKSUM_LENGTH = 4; - private static final int MAX_DATA_SIZE = 1024 * 1024; // 1MB + private static final int MAX_DATA_SIZE = 10 * 1024 * 1024; // 10MB @SuppressWarnings("serial") public static class MessageException extends Exception { @@ -78,9 +78,22 @@ public enum MessageType { ONLINE_ACCOUNTS(80), GET_ONLINE_ACCOUNTS(81), + ONLINE_ACCOUNTS_V2(82), + GET_ONLINE_ACCOUNTS_V2(83), ARBITRARY_DATA(90), - GET_ARBITRARY_DATA(91); + GET_ARBITRARY_DATA(91), + + BLOCKS(100), + GET_BLOCKS(101), + + ARBITRARY_DATA_FILE(110), + GET_ARBITRARY_DATA_FILE(111), + + ARBITRARY_DATA_FILE_LIST(120), + GET_ARBITRARY_DATA_FILE_LIST(121), + + ARBITRARY_SIGNATURES(130); public final int value; public final Method fromByteBufferMethod; @@ -160,80 +173,72 @@ public MessageType getType() { /** * Attempt to read a message from byte buffer. * - * @param byteBuffer + * @param readOnlyBuffer * @return null if no complete message can be read * @throws MessageException */ - public static Message fromByteBuffer(ByteBuffer byteBuffer) throws MessageException { + public static Message fromByteBuffer(ByteBuffer readOnlyBuffer) throws MessageException { try { - byteBuffer.flip(); - - ByteBuffer readBuffer = byteBuffer.asReadOnlyBuffer(); - // Read only enough bytes to cover Message "magic" preamble byte[] messageMagic = new byte[MAGIC_LENGTH]; - readBuffer.get(messageMagic); + readOnlyBuffer.get(messageMagic); if (!Arrays.equals(messageMagic, Network.getInstance().getMessageMagic())) // Didn't receive correct Message "magic" throw new MessageException("Received incorrect message 'magic'"); // Find supporting object - int typeValue = readBuffer.getInt(); + int typeValue = readOnlyBuffer.getInt(); MessageType messageType = MessageType.valueOf(typeValue); if (messageType == null) // Unrecognised message type throw new MessageException(String.format("Received unknown message type [%d]", typeValue)); // Optional message ID - byte hasId = readBuffer.get(); + byte hasId = readOnlyBuffer.get(); int id = -1; if (hasId != 0) { - id = readBuffer.getInt(); + id = readOnlyBuffer.getInt(); if (id <= 0) // Invalid ID throw new MessageException("Invalid negative ID"); } - int dataSize = readBuffer.getInt(); + int dataSize = readOnlyBuffer.getInt(); if (dataSize > MAX_DATA_SIZE) // Too large throw new MessageException(String.format("Declared data length %d larger than max allowed %d", dataSize, MAX_DATA_SIZE)); + // Don't have all the data yet? + if (dataSize > 0 && dataSize + CHECKSUM_LENGTH > readOnlyBuffer.remaining()) + return null; + ByteBuffer dataSlice = null; if (dataSize > 0) { byte[] expectedChecksum = new byte[CHECKSUM_LENGTH]; - readBuffer.get(expectedChecksum); + readOnlyBuffer.get(expectedChecksum); - // Remember this position in readBuffer so we can pass to Message subclass - dataSlice = readBuffer.slice(); - - // Consume data from buffer - byte[] data = new byte[dataSize]; - readBuffer.get(data); - - // We successfully read all the data bytes, so we can set limit on dataSlice + // Slice data in readBuffer so we can pass to Message subclass + dataSlice = readOnlyBuffer.slice(); dataSlice.limit(dataSize); // Test checksum - byte[] actualChecksum = generateChecksum(data); + byte[] actualChecksum = generateChecksum(dataSlice); if (!Arrays.equals(expectedChecksum, actualChecksum)) throw new MessageException("Message checksum incorrect"); - } - - Message message = messageType.fromByteBuffer(id, dataSlice); - // We successfully read a message, so bump byteBuffer's position to reflect this - byteBuffer.position(readBuffer.position()); + // Reset position after being consumed by generateChecksum + dataSlice.position(0); + // Update position in readOnlyBuffer + readOnlyBuffer.position(readOnlyBuffer.position() + dataSize); + } - return message; + return messageType.fromByteBuffer(id, dataSlice); } catch (BufferUnderflowException e) { // Not enough bytes to fully decode message... return null; - } finally { - byteBuffer.compact(); } } @@ -241,6 +246,10 @@ protected static byte[] generateChecksum(byte[] data) { return Arrays.copyOfRange(Crypto.digest(data), 0, CHECKSUM_LENGTH); } + protected static byte[] generateChecksum(ByteBuffer dataBuffer) { + return Arrays.copyOfRange(Crypto.digest(dataBuffer), 0, CHECKSUM_LENGTH); + } + public byte[] toBytes() throws MessageException { try { ByteArrayOutputStream bytes = new ByteArrayOutputStream(256); diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java index 75109a0a4..02c467179 100644 --- a/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java +++ b/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java @@ -6,6 +6,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import org.qortal.data.network.OnlineAccountData; import org.qortal.transform.Transformer; @@ -14,7 +15,7 @@ import com.google.common.primitives.Longs; public class OnlineAccountsMessage extends Message { - private static final int MAX_ACCOUNT_COUNT = 1000; + private static final int MAX_ACCOUNT_COUNT = 5000; private List onlineAccounts; @@ -25,7 +26,7 @@ public OnlineAccountsMessage(List onlineAccounts) { private OnlineAccountsMessage(int id, List onlineAccounts) { super(id, MessageType.ONLINE_ACCOUNTS); - this.onlineAccounts = onlineAccounts; + this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList()); } public List getOnlineAccounts() { @@ -35,12 +36,9 @@ public List getOnlineAccounts() { public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { final int accountCount = bytes.getInt(); - if (accountCount > MAX_ACCOUNT_COUNT) - return null; - List onlineAccounts = new ArrayList<>(accountCount); - for (int i = 0; i < accountCount; ++i) { + for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) { long timestamp = bytes.getLong(); byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java new file mode 100644 index 000000000..f0fce81e2 --- /dev/null +++ b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java @@ -0,0 +1,124 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.qortal.data.network.OnlineAccountData; +import org.qortal.transform.Transformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * For sending online accounts info to remote peer. + * + * Different format to V1: + * V1 is: number of entries, then timestamp + sig + pubkey for each entry + * V2 is: groups of: number of entries, timestamp, then sig + pubkey for each entry + * + * Also V2 only builds online accounts message once! + */ +public class OnlineAccountsV2Message extends Message { + private List onlineAccounts; + private byte[] cachedData; + + public OnlineAccountsV2Message(List onlineAccounts) { + this(-1, onlineAccounts); + } + + private OnlineAccountsV2Message(int id, List onlineAccounts) { + super(id, MessageType.ONLINE_ACCOUNTS_V2); + + this.onlineAccounts = onlineAccounts; + } + + public List getOnlineAccounts() { + return this.onlineAccounts; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + int accountCount = bytes.getInt(); + + List onlineAccounts = new ArrayList<>(accountCount); + + while (accountCount > 0) { + long timestamp = bytes.getLong(); + + for (int i = 0; i < accountCount; ++i) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; + bytes.get(signature); + + byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + bytes.get(publicKey); + + onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey)); + } + + if (bytes.hasRemaining()) { + accountCount = bytes.getInt(); + } else { + // we've finished + accountCount = 0; + } + } + + return new OnlineAccountsV2Message(id, onlineAccounts); + } + + @Override + protected synchronized byte[] toData() { + if (this.cachedData != null) + return this.cachedData; + + // Shortcut in case we have no online accounts + if (this.onlineAccounts.isEmpty()) { + this.cachedData = Ints.toByteArray(0); + return this.cachedData; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (int i = 0; i < this.onlineAccounts.size(); ++i) { + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + Long timestamp = onlineAccountData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + this.onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH); + + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (int i = 0; i < this.onlineAccounts.size(); ++i) { + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + + if (onlineAccountData.getTimestamp() == timestamp) { + bytes.write(onlineAccountData.getSignature()); + + bytes.write(onlineAccountData.getPublicKey()); + } + } + } + + this.cachedData = bytes.toByteArray(); + return this.cachedData; + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/payment/Payment.java b/src/main/java/org/qortal/payment/Payment.java index cd7f1118d..8b6070eea 100644 --- a/src/main/java/org/qortal/payment/Payment.java +++ b/src/main/java/org/qortal/payment/Payment.java @@ -40,8 +40,9 @@ public Payment(Repository repository) { public ValidationResult isValid(byte[] senderPublicKey, List payments, long fee, boolean isZeroAmountValid) throws DataException { AssetRepository assetRepository = this.repository.getAssetRepository(); - // Check fee is positive - if (fee <= 0) + // Check fee is positive or zero + // We have already checked that the fee is correct in the Transaction superclass + if (fee < 0) return ValidationResult.NEGATIVE_FEE; // Total up payment amounts by assetId diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index affbaf183..0f537ae94 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -1,9 +1,13 @@ package org.qortal.repository; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.List; +import java.util.Set; import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; +import org.qortal.utils.ByteArray; public interface ATRepository { @@ -15,12 +19,18 @@ public interface ATRepository { /** Returns where AT with passed address exists in repository */ public boolean exists(String atAddress) throws DataException; + /** Returns AT creator's public key, or null if not found */ + public byte[] getCreatorPublicKey(String atAddress) throws DataException; + /** Returns list of executable ATs, empty if none found */ public List getAllExecutableATs() throws DataException; /** Returns list of ATs with matching code hash, optionally executable only. */ public List getATsByFunctionality(byte[] codeHash, Boolean isExecutable, Integer limit, Integer offset, Boolean reverse) throws DataException; + /** Returns list of all ATs matching one of passed code hashes, optionally executable only. */ + public List getAllATsByFunctionality(Set codeHashes, Boolean isExecutable) throws DataException; + /** Returns creation block height given AT's address or null if not found */ public Integer getATCreationBlockHeight(String atAddress) throws DataException; @@ -54,10 +64,48 @@ public interface ATRepository { */ public ATStateData getLatestATState(String atAddress) throws DataException; + /** + * Returns final ATStateData for ATs matching codeHash (required) + * and specific data segment value (optional). + *

+ * If searching for specific data segment value, both dataByteOffset + * and expectedValue need to be non-null. + *

+ * Note that dataByteOffset starts from 0 and will typically be + * a multiple of MachineState.VALUE_SIZE, which is usually 8: + * width of a long. + *

+ * Although expectedValue, if provided, is natively an unsigned long, + * the data segment comparison is done via unsigned hex string. + */ + public List getMatchingFinalATStates(byte[] codeHash, Boolean isFinished, + Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight, + Integer limit, Integer offset, Boolean reverse) throws DataException; + + /** + * Returns final ATStateData for ATs matching codeHash (required) + * and specific data segment value (optional), returning at least + * minimumCount entries over a span of at least + * minimumPeriod ms, given enough entries in repository. + *

+ * If searching for specific data segment value, both dataByteOffset + * and expectedValue need to be non-null. + *

+ * Note that dataByteOffset starts from 0 and will typically be + * a multiple of MachineState.VALUE_SIZE, which is usually 8: + * width of a long. + *

+ * Although expectedValue, if provided, is natively an unsigned long, + * the data segment comparison is done via unsigned hex string. + */ + public List getMatchingFinalATStatesQuorum(byte[] codeHash, Boolean isFinished, + Integer dataByteOffset, Long expectedValue, + int minimumCount, int maximumCount, long minimumPeriod) throws DataException; + /** * Returns all ATStateData for a given block height. *

- * Unlike getATState, only returns ATStateData saved at the given height. + * Unlike getATState, only returns partial ATStateData saved at the given height. * * @param height * - block height @@ -66,6 +114,44 @@ public interface ATRepository { */ public List getBlockATStatesAtHeight(int height) throws DataException; + + /** Rebuild the latest AT states cache, necessary for AT state trimming/pruning. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ + public void rebuildLatestAtStates() throws DataException; + + + /** Returns height of first trimmable AT state. */ + public int getAtTrimHeight() throws DataException; + + /** Sets new base height for AT state trimming. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ + public void setAtTrimHeight(int trimHeight) throws DataException; + + /** Trims full AT state data between passed heights. Returns number of trimmed rows. */ + public int trimAtStates(int minHeight, int maxHeight, int limit) throws DataException; + + + /** Returns height of first prunable AT state. */ + public int getAtPruneHeight() throws DataException; + + /** Sets new base height for AT state pruning. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ + public void setAtPruneHeight(int pruneHeight) throws DataException; + + /** Prunes full AT state data between passed heights. Returns number of pruned rows. */ + public int pruneAtStates(int minHeight, int maxHeight) throws DataException; + + + /** Checks for the presence of the ATStatesHeightIndex in repository */ + public boolean hasAtStatesHeightIndex() throws DataException; + + /** * Save ATStateData into repository. *

@@ -88,4 +174,32 @@ public interface ATRepository { /** Delete state data for all ATs at this height */ public void deleteATStates(int height) throws DataException; + // Finding transactions for ATs to process + + static class NextTransactionInfo { + public final int height; + public final int sequence; + public final byte[] signature; + + public NextTransactionInfo(int height, int sequence, byte[] signature) { + this.height = height; + this.sequence = sequence; + this.signature = signature; + } + } + + /** + * Find next transaction for AT to process. + *

+ * @param recipient AT address + * @param height starting height + * @param sequence starting sequence + * @return next transaction info, or null if none found + */ + public NextTransactionInfo findNextTransaction(String recipient, int height, int sequence) throws DataException; + + // Other + + public void checkConsistency() throws DataException; + } diff --git a/src/main/java/org/qortal/repository/AccountRepository.java b/src/main/java/org/qortal/repository/AccountRepository.java index f6997fae3..256f9556e 100644 --- a/src/main/java/org/qortal/repository/AccountRepository.java +++ b/src/main/java/org/qortal/repository/AccountRepository.java @@ -4,6 +4,7 @@ import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; +import org.qortal.data.account.EligibleQoraHolderData; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.QortFromQoraData; import org.qortal.data.account.RewardShareData; @@ -89,6 +90,13 @@ public interface AccountRepository { */ public int modifyMintedBlockCount(String address, int delta) throws DataException; + /** + * Modifies batch of accounts' minted block count only. + *

+ * This is a one-shot, batch version of modifyMintedBlockCount(String, int) above. + */ + public void modifyMintedBlockCounts(List addresses, int delta) throws DataException; + /** Delete account from repository. */ public void delete(String address) throws DataException; @@ -106,6 +114,9 @@ public interface AccountRepository { */ public AccountBalanceData getBalance(String address, long assetId) throws DataException; + /** Returns all account balances for given assetID, optionally excluding zero balances. */ + public List getAssetBalances(long assetId, Boolean excludeZero) throws DataException; + /** How to order results when fetching asset balances. */ public enum BalanceOrdering { /** assetID first, then balance, then account address */ @@ -116,15 +127,18 @@ public enum BalanceOrdering { ASSET_ACCOUNT } - /** Returns all account balances for given assetID, optionally excluding zero balances. */ - public List getAssetBalances(long assetId, Boolean excludeZero) throws DataException; - /** Returns account balances for matching addresses / assetIDs, optionally excluding zero balances, with pagination, used by API. */ public List getAssetBalances(List addresses, List assetIds, BalanceOrdering balanceOrdering, Boolean excludeZero, Integer limit, Integer offset, Boolean reverse) throws DataException; /** Modifies account's asset balance by deltaBalance. */ public void modifyAssetBalance(String address, long assetId, long deltaBalance) throws DataException; + /** Modifies a batch of account asset balances, treating AccountBalanceData.balance as deltaBalance. */ + public void modifyAssetBalances(List accountBalanceDeltas) throws DataException; + + /** Batch update of account asset balances. */ + public void setAssetBalances(List accountBalances) throws DataException; + public void save(AccountBalanceData accountBalanceData) throws DataException; public void delete(String address, long assetId) throws DataException; @@ -156,6 +170,16 @@ public enum BalanceOrdering { */ public RewardShareData getRewardShareByIndex(int index) throws DataException; + /** + * Returns list of reward-share data using array of indexes into list of reward-shares (sorted by reward-share public key). + *

+ * This is a one-shot, batch form of the above getRewardShareByIndex(int) call. + * + * @return list of reward-share data, or null if one (or more) index is invalid + * @throws DataException + */ + public List getRewardSharesByIndexes(int[] indexes) throws DataException; + public boolean rewardShareExists(byte[] rewardSharePublicKey) throws DataException; public void save(RewardShareData rewardShareData) throws DataException; @@ -167,15 +191,17 @@ public enum BalanceOrdering { public List getMintingAccounts() throws DataException; + public MintingAccountData getMintingAccount(byte[] mintingAccountKey) throws DataException; + public void save(MintingAccountData mintingAccountData) throws DataException; - /** Delete minting account info, used by BlockMinter, from repository using passed private key. */ - public int delete(byte[] mintingAccountPrivateKey) throws DataException; + /** Delete minting account info, used by BlockMinter, from repository using passed public or private key. */ + public int delete(byte[] mintingAccountKey) throws DataException; // Managing QORT from legacy QORA /** - * Returns balance data for accounts with legacy QORA asset that are eligible + * Returns full info for accounts with legacy QORA asset that are eligible * for more block reward (block processing) or for block reward removal (block orphaning). *

* For block processing, accounts that have already received their final QORT reward for owning @@ -187,7 +213,7 @@ public enum BalanceOrdering { * @param blockHeight QORT reward must have be present at this height (for orphaning only) * @throws DataException */ - public List getEligibleLegacyQoraHolders(Integer blockHeight) throws DataException; + public List getEligibleLegacyQoraHolders(Integer blockHeight) throws DataException; public QortFromQoraData getQortFromQoraInfo(String address) throws DataException; diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 80f8c1e37..ea8ae0de2 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -1,6 +1,13 @@ package org.qortal.repository; +import org.qortal.arbitrary.misc.Service; +import org.qortal.data.arbitrary.ArbitraryResourceInfo; +import org.qortal.data.arbitrary.ArbitraryResourceNameInfo; +import org.qortal.data.network.ArbitraryPeerData; import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.*; + +import java.util.List; public interface ArbitraryRepository { @@ -12,4 +19,28 @@ public interface ArbitraryRepository { public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException; + public List getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException; + + public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException; + + + public List getArbitraryResources(Service service, String identifier, String name, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException; + + public List searchArbitraryResources(Service service, String query, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException; + + public List getArbitraryResourceCreatorNames(Service service, String identifier, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException; + + + public List getArbitraryPeerDataForSignature(byte[] signature) throws DataException; + + public ArbitraryPeerData getArbitraryPeerDataForSignatureAndPeer(byte[] signature, String peerAddress) throws DataException; + + public ArbitraryPeerData getArbitraryPeerDataForSignatureAndHost(byte[] signature, String host) throws DataException; + + public void save(ArbitraryPeerData arbitraryPeerData) throws DataException; + + public void delete(ArbitraryPeerData arbitraryPeerData) throws DataException; + + public void deleteArbitraryPeersWithSignature(byte[] signature) throws DataException; + } diff --git a/src/main/java/org/qortal/repository/BlockArchiveReader.java b/src/main/java/org/qortal/repository/BlockArchiveReader.java new file mode 100644 index 000000000..b6d7cdd6e --- /dev/null +++ b/src/main/java/org/qortal/repository/BlockArchiveReader.java @@ -0,0 +1,286 @@ +package org.qortal.repository; + +import com.google.common.primitives.Ints; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockArchiveData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.settings.Settings; +import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformer; +import org.qortal.utils.Triple; + +import static org.qortal.transform.Transformer.INT_LENGTH; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + +public class BlockArchiveReader { + + private static BlockArchiveReader instance; + private Map> fileListCache = Collections.synchronizedMap(new HashMap<>()); + + private static final Logger LOGGER = LogManager.getLogger(BlockArchiveReader.class); + + public BlockArchiveReader() { + + } + + public static synchronized BlockArchiveReader getInstance() { + if (instance == null) { + instance = new BlockArchiveReader(); + } + + return instance; + } + + private void fetchFileList() { + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath(); + File archiveDirFile = archivePath.toFile(); + String[] files = archiveDirFile.list(); + Map> map = new HashMap<>(); + + if (files != null) { + for (String file : files) { + Path filePath = Paths.get(file); + String filename = filePath.getFileName().toString(); + + // Parse the filename + if (filename == null || !filename.contains("-") || !filename.contains(".")) { + // Not a usable file + continue; + } + // Remove the extension and split into two parts + String[] parts = filename.substring(0, filename.lastIndexOf('.')).split("-"); + Integer startHeight = Integer.parseInt(parts[0]); + Integer endHeight = Integer.parseInt(parts[1]); + Integer range = endHeight - startHeight; + map.put(filename, new Triple(startHeight, endHeight, range)); + } + } + this.fileListCache = map; + } + + public Triple, List> fetchBlockAtHeight(int height) { + if (this.fileListCache.isEmpty()) { + this.fetchFileList(); + } + + byte[] serializedBytes = this.fetchSerializedBlockBytesForHeight(height); + if (serializedBytes == null) { + return null; + } + + ByteBuffer byteBuffer = ByteBuffer.wrap(serializedBytes); + Triple, List> blockInfo = null; + try { + blockInfo = BlockTransformer.fromByteBuffer(byteBuffer); + if (blockInfo != null && blockInfo.getA() != null) { + // Block height is stored outside of the main serialized bytes, so it + // won't be set automatically. + blockInfo.getA().setHeight(height); + } + } catch (TransformationException e) { + return null; + } + return blockInfo; + } + + public Triple, List> fetchBlockWithSignature( + byte[] signature, Repository repository) { + + if (this.fileListCache.isEmpty()) { + this.fetchFileList(); + } + + Integer height = this.fetchHeightForSignature(signature, repository); + if (height != null) { + return this.fetchBlockAtHeight(height); + } + return null; + } + + public List, List>> fetchBlocksFromRange( + int startHeight, int endHeight) { + + List, List>> blockInfoList = new ArrayList<>(); + + for (int height = startHeight; height <= endHeight; height++) { + Triple, List> blockInfo = this.fetchBlockAtHeight(height); + if (blockInfo == null) { + return blockInfoList; + } + blockInfoList.add(blockInfo); + } + return blockInfoList; + } + + public Integer fetchHeightForSignature(byte[] signature, Repository repository) { + // Lookup the height for the requested signature + try { + BlockArchiveData archivedBlock = repository.getBlockArchiveRepository().getBlockArchiveDataForSignature(signature); + if (archivedBlock == null) { + return null; + } + return archivedBlock.getHeight(); + + } catch (DataException e) { + return null; + } + } + + public int fetchHeightForTimestamp(long timestamp, Repository repository) { + // Lookup the height for the requested signature + try { + return repository.getBlockArchiveRepository().getHeightFromTimestamp(timestamp); + + } catch (DataException e) { + return 0; + } + } + + private String getFilenameForHeight(int height) { + synchronized (this.fileListCache) { + Iterator it = this.fileListCache.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry pair = (Map.Entry) it.next(); + if (pair == null && pair.getKey() == null && pair.getValue() == null) { + continue; + } + Triple heightInfo = (Triple) pair.getValue(); + Integer startHeight = heightInfo.getA(); + Integer endHeight = heightInfo.getB(); + + if (height >= startHeight && height <= endHeight) { + // Found the correct file + String filename = (String) pair.getKey(); + return filename; + } + } + } + + return null; + } + + public byte[] fetchSerializedBlockBytesForSignature(byte[] signature, boolean includeHeightPrefix, Repository repository) { + + if (this.fileListCache.isEmpty()) { + this.fetchFileList(); + } + + Integer height = this.fetchHeightForSignature(signature, repository); + if (height != null) { + byte[] blockBytes = this.fetchSerializedBlockBytesForHeight(height); + if (blockBytes == null) { + return null; + } + + // When responding to a peer with a BLOCK message, we must prefix the byte array with the block height + // This mimics the toData() method in BlockMessage and CachedBlockMessage + if (includeHeightPrefix) { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(blockBytes.length + INT_LENGTH); + try { + bytes.write(Ints.toByteArray(height)); + bytes.write(blockBytes); + return bytes.toByteArray(); + + } catch (IOException e) { + return null; + } + } + return blockBytes; + } + return null; + } + + public byte[] fetchSerializedBlockBytesForHeight(int height) { + String filename = this.getFilenameForHeight(height); + if (filename == null) { + // We don't have this block in the archive + // Invalidate the file list cache in case it is out of date + this.invalidateFileListCache(); + return null; + } + + Path filePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive", filename).toAbsolutePath(); + RandomAccessFile file = null; + try { + file = new RandomAccessFile(filePath.toString(), "r"); + // Get info about this file (the "fixed length header") + final int version = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + final int startHeight = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + final int endHeight = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + file.readInt(); // Block count (unused) // Do not remove or comment out, as it is moving the file pointer + final int variableHeaderLength = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + final int fixedHeaderLength = (int)file.getFilePointer(); + // End of fixed length header + + // Make sure the version is one we recognize + if (version != 1) { + LOGGER.info("Error: unknown version in file {}: {}", filename, version); + return null; + } + + // Verify that the block is within the reported range + if (height < startHeight || height > endHeight) { + LOGGER.info("Error: requested height {} but the range of file {} is {}-{}", + height, filename, startHeight, endHeight); + return null; + } + + // Seek to the location of the block index in the variable length header + final int locationOfBlockIndexInVariableHeaderSegment = (height - startHeight) * INT_LENGTH; + file.seek(fixedHeaderLength + locationOfBlockIndexInVariableHeaderSegment); + + // Read the value to obtain the index of this block in the data segment + int locationOfBlockInDataSegment = file.readInt(); + + // Now seek to the block data itself + int dataSegmentStartIndex = fixedHeaderLength + variableHeaderLength + INT_LENGTH; // Confirmed correct + file.seek(dataSegmentStartIndex + locationOfBlockInDataSegment); + + // Read the block metadata + int blockHeight = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + int blockLength = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + + // Ensure the block height matches the one requested + if (blockHeight != height) { + LOGGER.info("Error: height {} does not match requested: {}", blockHeight, height); + return null; + } + + // Now retrieve the block's serialized bytes + byte[] blockBytes = new byte[blockLength]; + file.read(blockBytes); + + return blockBytes; + + } catch (FileNotFoundException e) { + LOGGER.info("File {} not found: {}", filename, e.getMessage()); + return null; + } catch (IOException e) { + LOGGER.info("Unable to read block {} from archive: {}", height, e.getMessage()); + return null; + } + finally { + // Close the file + if (file != null) { + try { + file.close(); + } catch (IOException e) { + // Failed to close, but no need to handle this + } + } + } + } + + public void invalidateFileListCache() { + this.fileListCache.clear(); + } + +} diff --git a/src/main/java/org/qortal/repository/BlockArchiveRepository.java b/src/main/java/org/qortal/repository/BlockArchiveRepository.java new file mode 100644 index 000000000..9859e1378 --- /dev/null +++ b/src/main/java/org/qortal/repository/BlockArchiveRepository.java @@ -0,0 +1,139 @@ +package org.qortal.repository; + +import org.qortal.api.model.BlockSignerSummary; +import org.qortal.data.block.BlockArchiveData; +import org.qortal.data.block.BlockData; +import org.qortal.data.block.BlockSummaryData; + +import java.util.List; + +public interface BlockArchiveRepository { + + /** + * Returns BlockData from archive using block signature. + * + * @param signature + * @return block data, or null if not found in archive. + * @throws DataException + */ + public BlockData fromSignature(byte[] signature) throws DataException; + + /** + * Return height of block in archive using block's signature. + * + * @param signature + * @return height, or 0 if not found in blockchain. + * @throws DataException + */ + public int getHeightFromSignature(byte[] signature) throws DataException; + + /** + * Returns BlockData from archive using block height. + * + * @param height + * @return block data, or null if not found in blockchain. + * @throws DataException + */ + public BlockData fromHeight(int height) throws DataException; + + /** + * Returns a list of BlockData objects from archive using + * block height range. + * + * @param startHeight + * @return a list of BlockData objects, or an empty list if + * not found in blockchain. It is not guaranteed that all + * requested blocks will be returned. + * @throws DataException + */ + public List fromRange(int startHeight, int endHeight) throws DataException; + + /** + * Returns BlockData from archive using block reference. + * Currently relies on a child block being the one block + * higher than its parent. This limitation can be removed + * by storing the reference in the BlockArchive table, but + * this has been avoided to reduce space. + * + * @param reference + * @return block data, or null if either parent or child + * not found in the archive. + * @throws DataException + */ + public BlockData fromReference(byte[] reference) throws DataException; + + /** + * Return height of block with timestamp just before passed timestamp. + * + * @param timestamp + * @return height, or 0 if not found in blockchain. + * @throws DataException + */ + public int getHeightFromTimestamp(long timestamp) throws DataException; + + /** + * Returns block timestamp for a given height. + * + * @param height + * @return timestamp, or 0 if height is out of bounds. + * @throws DataException + */ + public long getTimestampFromHeight(int height) throws DataException; + + /** + * Returns block summaries for blocks signed by passed public key, or reward-share with minter with passed public key. + */ + public List getBlockSummariesBySigner(byte[] signerPublicKey, Integer limit, Integer offset, Boolean reverse) throws DataException; + + /** + * Returns summaries of block signers, optionally limited to passed addresses. + * This combines both the BlockArchive and the Blocks data into a single result set. + */ + public List getBlockSigners(List addresses, Integer limit, Integer offset, Boolean reverse) throws DataException; + + + /** Returns height of first unarchived block. */ + public int getBlockArchiveHeight() throws DataException; + + /** Sets new height for block archiving. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ + public void setBlockArchiveHeight(int archiveHeight) throws DataException; + + + /** + * Returns the block archive data for a given signature, from the block archive. + *

+ * This method will return null if no block archive has been built for the + * requested signature. In those cases, the height (and other data) can be + * looked up using the Blocks table. This allows a block to be located in + * the archive when we only know its signature. + *

+ * + * @param signature + * @throws DataException + */ + public BlockArchiveData getBlockArchiveDataForSignature(byte[] signature) throws DataException; + + /** + * Saves a block archive entry into the repository. + *

+ * This can be used to find the height of a block by its signature, without + * having access to the block data itself. + *

+ * + * @param blockArchiveData + * @throws DataException + */ + public void save(BlockArchiveData blockArchiveData) throws DataException; + + /** + * Deletes a block archive entry from the repository. + * + * @param blockArchiveData + * @throws DataException + */ + public void delete(BlockArchiveData blockArchiveData) throws DataException; + +} diff --git a/src/main/java/org/qortal/repository/BlockArchiveWriter.java b/src/main/java/org/qortal/repository/BlockArchiveWriter.java new file mode 100644 index 000000000..5127bf9b3 --- /dev/null +++ b/src/main/java/org/qortal/repository/BlockArchiveWriter.java @@ -0,0 +1,202 @@ +package org.qortal.repository; + +import com.google.common.primitives.Ints; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.block.Block; +import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; +import org.qortal.data.block.BlockArchiveData; +import org.qortal.data.block.BlockData; +import org.qortal.settings.Settings; +import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformer; + +import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class BlockArchiveWriter { + + public enum BlockArchiveWriteResult { + OK, + STOPPING, + NOT_ENOUGH_BLOCKS, + BLOCK_NOT_FOUND + } + + private static final Logger LOGGER = LogManager.getLogger(BlockArchiveWriter.class); + + public static final long DEFAULT_FILE_SIZE_TARGET = 100 * 1024 * 1024; // 100MiB + + private int startHeight; + private final int endHeight; + private final Repository repository; + + private long fileSizeTarget = DEFAULT_FILE_SIZE_TARGET; + private boolean shouldEnforceFileSizeTarget = true; + + private int writtenCount; + private int lastWrittenHeight; + private Path outputPath; + + public BlockArchiveWriter(int startHeight, int endHeight, Repository repository) { + this.startHeight = startHeight; + this.endHeight = endHeight; + this.repository = repository; + } + + public static int getMaxArchiveHeight(Repository repository) throws DataException { + // We must only archive trimmed blocks, or the archive will grow far too large + final int accountSignaturesTrimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight(); + final int atTrimStartHeight = repository.getATRepository().getAtTrimHeight(); + final int trimStartHeight = Math.min(accountSignaturesTrimStartHeight, atTrimStartHeight); + return trimStartHeight - 1; // subtract 1 because these values represent the first _untrimmed_ block + } + + public static boolean isArchiverUpToDate(Repository repository) throws DataException { + final int maxArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + final int actualArchiveHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); + final float progress = (float)actualArchiveHeight / (float) maxArchiveHeight; + LOGGER.debug(String.format("maxArchiveHeight: %d, actualArchiveHeight: %d, progress: %f", + maxArchiveHeight, actualArchiveHeight, progress)); + + // If archiver is within 95% of the maximum, treat it as up to date + // We need several percent as an allowance because the archiver will only + // save files when they reach the target size + return (progress >= 0.95); + } + + public BlockArchiveWriteResult write() throws DataException, IOException, TransformationException, InterruptedException { + // Create the archive folder if it doesn't exist + // This is a subfolder of the db directory, to make bootstrapping easier + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath(); + try { + Files.createDirectories(archivePath); + } catch (IOException e) { + LOGGER.info("Unable to create archive folder"); + throw new DataException("Unable to create archive folder"); + } + + // Determine start height of blocks to fetch + if (startHeight <= 2) { + // Skip genesis block, as it's not designed to be transmitted, and we can build that from blockchain.json + // TODO: include genesis block if we can + startHeight = 2; + } + + // Header bytes will store the block indexes + ByteArrayOutputStream headerBytes = new ByteArrayOutputStream(); + // Bytes will store the actual block data + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + LOGGER.info(String.format("Fetching blocks from height %d...", startHeight)); + int i = 0; + while (headerBytes.size() + bytes.size() < this.fileSizeTarget + || this.shouldEnforceFileSizeTarget == false) { + + if (Controller.isStopping()) { + return BlockArchiveWriteResult.STOPPING; + } + if (Synchronizer.getInstance().isSynchronizing()) { + continue; + } + + int currentHeight = startHeight + i; + if (currentHeight > endHeight) { + break; + } + + //LOGGER.info("Fetching block {}...", currentHeight); + + BlockData blockData = repository.getBlockRepository().fromHeight(currentHeight); + if (blockData == null) { + return BlockArchiveWriteResult.BLOCK_NOT_FOUND; + } + + // Write the signature and height into the BlockArchive table + BlockArchiveData blockArchiveData = new BlockArchiveData(blockData); + repository.getBlockArchiveRepository().save(blockArchiveData); + repository.saveChanges(); + + // Write the block data to some byte buffers + Block block = new Block(repository, blockData); + int blockIndex = bytes.size(); + // Write block index to header + headerBytes.write(Ints.toByteArray(blockIndex)); + // Write block height + bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); + byte[] blockBytes = BlockTransformer.toBytes(block); + // Write block length + bytes.write(Ints.toByteArray(blockBytes.length)); + // Write block bytes + bytes.write(blockBytes); + i++; + + } + int totalLength = headerBytes.size() + bytes.size(); + LOGGER.info(String.format("Total length of %d blocks is %d bytes", i, totalLength)); + + // Validate file size, in case something went wrong + if (totalLength < fileSizeTarget && this.shouldEnforceFileSizeTarget) { + return BlockArchiveWriteResult.NOT_ENOUGH_BLOCKS; + } + + // We have enough blocks to create a new file + int endHeight = startHeight + i - 1; + int version = 1; + String filePath = String.format("%s/%d-%d.dat", archivePath.toString(), startHeight, endHeight); + FileOutputStream fileOutputStream = new FileOutputStream(filePath); + // Write version number + fileOutputStream.write(Ints.toByteArray(version)); + // Write start height + fileOutputStream.write(Ints.toByteArray(startHeight)); + // Write end height + fileOutputStream.write(Ints.toByteArray(endHeight)); + // Write total count + fileOutputStream.write(Ints.toByteArray(i)); + // Write dynamic header (block indexes) segment length + fileOutputStream.write(Ints.toByteArray(headerBytes.size())); + // Write dynamic header (block indexes) data + headerBytes.writeTo(fileOutputStream); + // Write data segment (block data) length + fileOutputStream.write(Ints.toByteArray(bytes.size())); + // Write data + bytes.writeTo(fileOutputStream); + // Close the file + fileOutputStream.close(); + + // Invalidate cache so that the rest of the app picks up the new file + BlockArchiveReader.getInstance().invalidateFileListCache(); + + this.writtenCount = i; + this.lastWrittenHeight = endHeight; + this.outputPath = Paths.get(filePath); + return BlockArchiveWriteResult.OK; + } + + public int getWrittenCount() { + return this.writtenCount; + } + + public int getLastWrittenHeight() { + return this.lastWrittenHeight; + } + + public Path getOutputPath() { + return this.outputPath; + } + + public void setFileSizeTarget(long fileSizeTarget) { + this.fileSizeTarget = fileSizeTarget; + } + + // For testing, to avoid having to pre-calculate file sizes + public void setShouldEnforceFileSizeTarget(boolean shouldEnforceFileSizeTarget) { + this.shouldEnforceFileSizeTarget = shouldEnforceFileSizeTarget; + } + +} diff --git a/src/main/java/org/qortal/repository/BlockRepository.java b/src/main/java/org/qortal/repository/BlockRepository.java index 243722129..76891c368 100644 --- a/src/main/java/org/qortal/repository/BlockRepository.java +++ b/src/main/java/org/qortal/repository/BlockRepository.java @@ -60,6 +60,15 @@ public interface BlockRepository { */ public int getHeightFromTimestamp(long timestamp) throws DataException; + /** + * Returns block timestamp for a given height. + * + * @param height + * @return timestamp, or 0 if height is out of bounds. + * @throws DataException + */ + public long getTimestampFromHeight(int height) throws DataException; + /** * Return highest block height from repository. * @@ -128,20 +137,43 @@ public default List getTransactionsFromSignature(byte[] signatu */ public List getBlockSummaries(int firstBlockHeight, int lastBlockHeight) throws DataException; + /** Returns height of first trimmable online accounts signatures. */ + public int getOnlineAccountsSignaturesTrimHeight() throws DataException; + + /** Sets new base height for trimming online accounts signatures. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ + public void setOnlineAccountsSignaturesTrimHeight(int trimHeight) throws DataException; + /** - * Trim online accounts signatures from blocks older than passed timestamp. + * Trim online accounts signatures from blocks between passed heights. * - * @param timestamp * @return number of blocks trimmed */ - public int trimOldOnlineAccountsSignatures(long timestamp) throws DataException; + public int trimOldOnlineAccountsSignatures(int minHeight, int maxHeight) throws DataException; /** - * Returns first (lowest height) block that doesn't link back to genesis block. + * Returns first (lowest height) block that doesn't link back to specified block. * + * @param startHeight height of specified block * @throws DataException */ - public BlockData getDetachedBlockSignature() throws DataException; + public BlockData getDetachedBlockSignature(int startHeight) throws DataException; + + + /** Returns height of first prunable block. */ + public int getBlockPruneHeight() throws DataException; + + /** Sets new base height for block pruning. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ + public void setBlockPruneHeight(int pruneHeight) throws DataException; + + /** Prunes full block data between passed heights. Returns number of pruned rows. */ + public int pruneBlocks(int minHeight, int maxHeight) throws DataException; + /** * Saves block into repository. diff --git a/src/main/java/org/qortal/repository/Bootstrap.java b/src/main/java/org/qortal/repository/Bootstrap.java new file mode 100644 index 000000000..54736bc7f --- /dev/null +++ b/src/main/java/org/qortal/repository/Bootstrap.java @@ -0,0 +1,513 @@ +package org.qortal.repository; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.block.BlockChain; +import org.qortal.controller.Controller; +import org.qortal.crypto.Crypto; +import org.qortal.data.account.MintingAccountData; +import org.qortal.data.block.BlockData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.gui.SplashFrame; +import org.qortal.network.Network; +import org.qortal.repository.hsqldb.HSQLDBImportExport; +import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; +import org.qortal.settings.Settings; +import org.qortal.utils.NTP; +import org.qortal.utils.SevenZ; + +import java.io.BufferedInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.*; +import java.security.SecureRandom; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.ReentrantLock; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + + +public class Bootstrap { + + private Repository repository; + + private int retryMinutes = 1; + + private static final Logger LOGGER = LogManager.getLogger(Bootstrap.class); + + /** The maximum number of untrimmed blocks allowed to be included in a bootstrap, beyond the trim threshold */ + private static final int MAXIMUM_UNTRIMMED_BLOCKS = 100; + + /** The maximum number of unpruned blocks allowed to be included in a bootstrap, beyond the prune threshold */ + private static final int MAXIMUM_UNPRUNED_BLOCKS = 100; + + + public Bootstrap() { + } + + public Bootstrap(Repository repository) { + this.repository = repository; + } + + /** + * canCreateBootstrap() + * Performs basic initial checks to ensure everything is in order + * @return true if ready for bootstrap creation, or an exception if not + * All failure reasons are logged and included in the exception + * @throws DataException + */ + public boolean checkRepositoryState() throws DataException { + LOGGER.info("Checking repository state..."); + + final boolean isTopOnly = Settings.getInstance().isTopOnly(); + final boolean archiveEnabled = Settings.getInstance().isArchiveEnabled(); + + // Make sure we have a repository instance + if (repository == null) { + throw new DataException("Repository instance required to check if we can create a bootstrap."); + } + + // Require that a block archive has been built + if (!isTopOnly && !archiveEnabled) { + throw new DataException("Unable to create bootstrap because the block archive isn't enabled. " + + "Set {\"archivedEnabled\": true} in settings.json to fix."); + } + + // Make sure that the block archiver is up to date + boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository); + if (!upToDate) { + throw new DataException("Unable to create bootstrap because the block archive isn't fully built yet."); + } + + // Ensure that this database contains the ATStatesHeightIndex which was missing in some cases + boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex(); + if (!hasAtStatesHeightIndex) { + throw new DataException("Unable to create bootstrap due to missing ATStatesHeightIndex. A re-sync from genesis is needed."); + } + + // Ensure we have synced NTP time + if (NTP.getTime() == null) { + throw new DataException("Unable to create bootstrap because the node hasn't synced its time yet."); + } + + // Ensure the chain is synced + final BlockData chainTip = Controller.getInstance().getChainTip(); + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) { + throw new DataException("Unable to create bootstrap because the blockchain isn't fully synced."); + } + + // FUTURE: ensure trim and prune settings are using default values + + if (!isTopOnly) { + // We don't trim in top-only mode because we prune the blocks instead + // If we're not in top-only mode we should make sure that trimming is up to date + + // Ensure that the online account signatures have been fully trimmed + final int accountsTrimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight(); + final long accountsUpperTrimmableTimestamp = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime(); + final int accountsUpperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(accountsUpperTrimmableTimestamp); + final int accountsBlocksRemaining = accountsUpperTrimmableHeight - accountsTrimStartHeight; + if (accountsBlocksRemaining > MAXIMUM_UNTRIMMED_BLOCKS) { + throw new DataException(String.format("Blockchain is not fully trimmed. Please allow the node to run for longer, " + + "then try again. Blocks remaining (online accounts signatures): %d", accountsBlocksRemaining)); + } + + // Ensure that the AT states data has been fully trimmed + final int atTrimStartHeight = repository.getATRepository().getAtTrimHeight(); + final long atUpperTrimmableTimestamp = chainTip.getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime(); + final int atUpperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(atUpperTrimmableTimestamp); + final int atBlocksRemaining = atUpperTrimmableHeight - atTrimStartHeight; + if (atBlocksRemaining > MAXIMUM_UNTRIMMED_BLOCKS) { + throw new DataException(String.format("Blockchain is not fully trimmed. Please allow the node to run " + + "for longer, then try again. Blocks remaining (AT states): %d", atBlocksRemaining)); + } + } + + // Ensure that blocks have been fully pruned + final int blockPruneStartHeight = repository.getBlockRepository().getBlockPruneHeight(); + int blockUpperPrunableHeight = chainTip.getHeight() - Settings.getInstance().getPruneBlockLimit(); + if (archiveEnabled) { + blockUpperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1; + } + final int blocksPruneRemaining = blockUpperPrunableHeight - blockPruneStartHeight; + if (blocksPruneRemaining > MAXIMUM_UNPRUNED_BLOCKS) { + throw new DataException(String.format("Blockchain is not fully pruned. Please allow the node to run " + + "for longer, then try again. Blocks remaining: %d", blocksPruneRemaining)); + } + + // Ensure that AT states have been fully pruned + final int atPruneStartHeight = repository.getATRepository().getAtPruneHeight(); + int atUpperPrunableHeight = chainTip.getHeight() - Settings.getInstance().getPruneBlockLimit(); + if (archiveEnabled) { + atUpperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1; + } + final int atPruneRemaining = atUpperPrunableHeight - atPruneStartHeight; + if (atPruneRemaining > MAXIMUM_UNPRUNED_BLOCKS) { + throw new DataException(String.format("Blockchain is not fully pruned. Please allow the node to run " + + "for longer, then try again. Blocks remaining (AT states): %d", atPruneRemaining)); + } + + LOGGER.info("Repository state checks passed"); + return true; + } + + /** + * validateBlockchain + * Performs quick validation of recent blocks in blockchain, prior to creating a bootstrap + * @return true if valid, an exception if not + * @throws DataException + */ + public boolean validateBlockchain() throws DataException { + LOGGER.info("Validating blockchain..."); + + try { + BlockChain.validate(); + + LOGGER.info("Blockchain is valid"); + + return true; + } catch (DataException e) { + throw new DataException(String.format("Blockchain validation failed: %s", e.getMessage())); + } + } + + /** + * validateCompleteBlockchain + * Performs intensive validation of all blocks in blockchain + * @return true if valid, false if not + */ + public boolean validateCompleteBlockchain() { + LOGGER.info("Validating blockchain..."); + + try { + // Perform basic startup validation + BlockChain.validate(); + + // Perform more intensive full-chain validation + BlockChain.validateAllBlocks(); + + LOGGER.info("Blockchain is valid"); + + return true; + } catch (DataException e) { + LOGGER.info("Blockchain validation failed: {}", e.getMessage()); + return false; + } + } + + public String create() throws DataException, InterruptedException, IOException { + + // Make sure we have a repository instance + if (repository == null) { + throw new DataException("Repository instance required in order to create a boostrap"); + } + + LOGGER.info("Deleting temp directory if it exists..."); + this.deleteAllTempDirectories(); + + LOGGER.info("Acquiring blockchain lock..."); + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + blockchainLock.lockInterruptibly(); + + Path inputPath = null; + Path outputPath = null; + + try { + + LOGGER.info("Exporting local data..."); + repository.exportNodeLocalData(); + + LOGGER.info("Deleting trade bot states..."); + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + for (TradeBotData tradeBotData : allTradeBotData) { + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + } + + LOGGER.info("Deleting minting accounts..."); + List mintingAccounts = repository.getAccountRepository().getMintingAccounts(); + for (MintingAccountData mintingAccount : mintingAccounts) { + repository.getAccountRepository().delete(mintingAccount.getPrivateKey()); + } + + repository.saveChanges(); + + LOGGER.info("Deleting peers list..."); + repository.getNetworkRepository().deleteAllPeers(); + repository.saveChanges(); + + LOGGER.info("Adding initial peers..."); + Network.installInitialPeers(repository); + + LOGGER.info("Creating bootstrap..."); + // Timeout if the database isn't ready for backing up after 10 seconds + long timeout = 10 * 1000L; + repository.backup(false, "bootstrap", timeout); + + LOGGER.info("Moving files to output directory..."); + inputPath = Paths.get(Settings.getInstance().getRepositoryPath(), "bootstrap"); + outputPath = Paths.get(this.createTempDirectory().toString(), "bootstrap"); + + + // Move the db backup to a "bootstrap" folder in the root directory + Files.move(inputPath, outputPath, REPLACE_EXISTING); + + // If in archive mode, copy the archive folder to inside the bootstrap folder + if (!Settings.getInstance().isTopOnly() && Settings.getInstance().isArchiveEnabled()) { + FileUtils.copyDirectory( + Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toFile(), + Paths.get(outputPath.toString(), "archive").toFile() + ); + } + + LOGGER.info("Preparing output path..."); + Path compressedOutputPath = this.getBootstrapOutputPath(); + try { + Files.delete(compressedOutputPath); + } catch (NoSuchFileException e) { + // Doesn't exist, so no need to delete + } + + LOGGER.info("Compressing..."); + SevenZ.compress(compressedOutputPath.toString(), outputPath.toFile()); + + LOGGER.info("Generating checksum file..."); + String checksum = Crypto.digestHexString(compressedOutputPath.toFile(), 1024*1024); + Path checksumPath = Paths.get(String.format("%s.sha256", compressedOutputPath.toString())); + Files.writeString(checksumPath, checksum, StandardOpenOption.CREATE); + + // Return the path to the compressed bootstrap file + LOGGER.info("Bootstrap creation complete. Output file: {}", compressedOutputPath.toAbsolutePath().toString()); + return compressedOutputPath.toAbsolutePath().toString(); + + } + catch (TimeoutException e) { + throw new DataException(String.format("Unable to create bootstrap due to timeout: %s", e.getMessage())); + } + finally { + try { + LOGGER.info("Re-importing local data..."); + Path exportPath = HSQLDBImportExport.getExportDirectory(false); + repository.importDataFromFile(Paths.get(exportPath.toString(), "TradeBotStates.json").toString()); + repository.importDataFromFile(Paths.get(exportPath.toString(), "MintingAccounts.json").toString()); + repository.saveChanges(); + + } catch (IOException e) { + LOGGER.info("Unable to re-import local data, but created bootstrap is still valid. {}", e); + } + + LOGGER.info("Unlocking blockchain..."); + blockchainLock.unlock(); + + // Cleanup + LOGGER.info("Cleaning up..."); + Thread.sleep(5000L); + this.deleteAllTempDirectories(); + } + } + + public void startImport() throws InterruptedException { + while (!Controller.isStopping()) { + try (final Repository repository = RepositoryManager.getRepository()) { + this.repository = repository; + + this.updateStatus("Starting import of bootstrap..."); + + this.doImport(); + break; + + } catch (DataException e) { + LOGGER.info("Bootstrap import failed", e); + this.updateStatus(String.format("Bootstrapping failed. Retrying in %d minutes...", retryMinutes)); + Thread.sleep(retryMinutes * 60 * 1000L); + retryMinutes *= 2; + } + } + } + + private void doImport() throws DataException { + Path path = null; + try { + Path tempDir = this.createTempDirectory(); + String filename = String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), this.getFilename()); + path = Paths.get(tempDir.toString(), filename); + + this.downloadToPath(path); + this.importFromPath(path); + + } catch (InterruptedException | DataException | IOException e) { + throw new DataException("Unable to import bootstrap", e); + } + finally { + if (path != null) { + try { + Files.delete(path); + + } catch (IOException e) { + // Temp folder will be cleaned up below, so ignore this failure + } + } + this.deleteAllTempDirectories(); + } + } + + private String getFilename() { + boolean isTopOnly = Settings.getInstance().isTopOnly(); + boolean archiveEnabled = Settings.getInstance().isArchiveEnabled(); + boolean isTestnet = Settings.getInstance().isTestNet(); + String prefix = isTestnet ? "testnet-" : ""; + + if (isTopOnly) { + return prefix.concat("bootstrap-toponly.7z"); + } + else if (archiveEnabled) { + return prefix.concat("bootstrap-archive.7z"); + } + else { + return prefix.concat("bootstrap-full.7z"); + } + } + + private void downloadToPath(Path path) throws DataException { + String bootstrapHost = this.getRandomHost(); + String bootstrapFilename = this.getFilename(); + String bootstrapUrl = String.format("%s/%s", bootstrapHost, bootstrapFilename); + String type = Settings.getInstance().isTopOnly() ? "top-only" : "full node"; + + SplashFrame.getInstance().updateStatus(String.format("Downloading %s bootstrap...", type)); + LOGGER.info(String.format("Downloading %s bootstrap from %s ...", type, bootstrapUrl)); + + // Delete an existing file if it exists + try { + Files.delete(path); + } catch (IOException e) { + // No need to do anything + } + + // Get the total file size + URL url; + long fileSize; + try { + url = new URL(bootstrapUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("HEAD"); + connection.connect(); + fileSize = connection.getContentLengthLong(); + connection.disconnect(); + + } catch (MalformedURLException e) { + throw new DataException(String.format("Malformed URL when downloading bootstrap: %s", e.getMessage())); + } catch (IOException e) { + throw new DataException(String.format("Unable to get bootstrap file size from %s. " + + "Please check your internet connection.", e.getMessage())); + } + + // Download the file and update the status with progress + try (BufferedInputStream in = new BufferedInputStream(url.openStream()); + FileOutputStream fileOutputStream = new FileOutputStream(path.toFile())) { + byte[] buffer = new byte[1024 * 1024]; + long downloaded = 0; + int bytesRead; + while ((bytesRead = in.read(buffer, 0, 1024)) != -1) { + fileOutputStream.write(buffer, 0, bytesRead); + downloaded += bytesRead; + + if (fileSize > 0) { + int progress = (int)((double)downloaded / (double)fileSize * 100); + SplashFrame.getInstance().updateStatus(String.format("Downloading %s bootstrap... (%d%%)", type, progress)); + } + } + + } catch (IOException e) { + throw new DataException(String.format("Unable to download bootstrap: %s", e.getMessage())); + } + } + + public String getRandomHost() { + // Select a random host from bootstrapHosts + String[] hosts = Settings.getInstance().getBootstrapHosts(); + int index = new SecureRandom().nextInt(hosts.length); + String bootstrapHost = hosts[index]; + return bootstrapHost; + } + + public void importFromPath(Path path) throws InterruptedException, DataException, IOException { + + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + blockchainLock.lockInterruptibly(); + + try { + this.updateStatus("Stopping repository..."); + // Close the repository while we are still able to + // Otherwise, the caller will run into difficulties when it tries to close it + repository.discardChanges(); + repository.close(); + // Now close the repository factory so that we can swap out the database files + RepositoryManager.closeRepositoryFactory(); + + this.updateStatus("Deleting existing repository..."); + Path input = path.toAbsolutePath(); + Path output = path.toAbsolutePath().getParent().toAbsolutePath(); + Path inputPath = Paths.get(output.toString(), "bootstrap"); + Path outputPath = Paths.get(Settings.getInstance().getRepositoryPath()); + FileUtils.deleteDirectory(outputPath.toFile()); + + this.updateStatus("Extracting bootstrap..."); + SevenZ.decompress(input.toString(), output.toFile()); + + if (!inputPath.toFile().exists()) { + throw new DataException("Extracted bootstrap doesn't exist"); + } + + // Move the "bootstrap" folder in place of the "db" folder + this.updateStatus("Moving files to output directory..."); + Files.move(inputPath, outputPath); + + this.updateStatus("Starting repository from bootstrap..."); + } + finally { + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); + + blockchainLock.unlock(); + } + } + + private Path createTempDirectory() throws IOException { + Path initialPath = Paths.get(Settings.getInstance().getRepositoryPath()).toAbsolutePath().getParent(); + String baseDir = Paths.get(initialPath.toString(), "tmp").toFile().getCanonicalPath(); + String identifier = UUID.randomUUID().toString(); + Path tempDir = Paths.get(baseDir, identifier); + Files.createDirectories(tempDir); + return tempDir; + } + + private void deleteAllTempDirectories() { + Path initialPath = Paths.get(Settings.getInstance().getRepositoryPath()).toAbsolutePath().getParent(); + Path path = Paths.get(initialPath.toString(), "tmp"); + try { + FileUtils.deleteDirectory(path.toFile()); + } catch (IOException e) { + LOGGER.info("Unable to delete temp directory path: {}", path.toString(), e); + } + } + + public Path getBootstrapOutputPath() { + Path initialPath = Paths.get(Settings.getInstance().getRepositoryPath()).toAbsolutePath().getParent(); + String compressedFilename = String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), this.getFilename()); + Path compressedOutputPath = Paths.get(initialPath.toString(), compressedFilename); + return compressedOutputPath; + } + + private void updateStatus(String text) { + LOGGER.info(text); + SplashFrame.getInstance().updateStatus(text); + } + +} diff --git a/src/main/java/org/qortal/repository/CrossChainRepository.java b/src/main/java/org/qortal/repository/CrossChainRepository.java new file mode 100644 index 000000000..70ebdbf9a --- /dev/null +++ b/src/main/java/org/qortal/repository/CrossChainRepository.java @@ -0,0 +1,21 @@ +package org.qortal.repository; + +import java.util.List; + +import org.qortal.data.crosschain.TradeBotData; + +public interface CrossChainRepository { + + public TradeBotData getTradeBotData(byte[] tradePrivateKey) throws DataException; + + /** Returns true if there is an existing trade-bot entry relating to given AT address, excluding trade-bot entries with given states. */ + public boolean existsTradeWithAtExcludingStates(String atAddress, List excludeStates) throws DataException; + + public List getAllTradeBotData() throws DataException; + + public void save(TradeBotData tradeBotData) throws DataException; + + /** Delete trade-bot states using passed private key. */ + public int delete(byte[] tradePrivateKey) throws DataException; + +} diff --git a/src/main/java/org/qortal/repository/MessageRepository.java b/src/main/java/org/qortal/repository/MessageRepository.java new file mode 100644 index 000000000..db74f0e6e --- /dev/null +++ b/src/main/java/org/qortal/repository/MessageRepository.java @@ -0,0 +1,31 @@ +package org.qortal.repository; + +import java.util.List; + +import org.qortal.data.transaction.MessageTransactionData; + +public interface MessageRepository { + + /** + * Returns list of confirmed MESSAGE transaction data matching (some) participants. + *

+ * At least one of senderPublicKey or recipient must be specified. + *

+ * @throws DataException + */ + public List getMessagesByParticipants(byte[] senderPublicKey, + String recipient, Integer limit, Integer offset, Boolean reverse) throws DataException; + + /** + * Does a MESSAGE exist with matching sender (pubkey), recipient and message payload? + *

+ * Includes both confirmed and unconfirmed transactions! + *

+ * @param senderPublicKey + * @param recipient + * @param messageData + * @return true if a message exists, false otherwise + */ + public boolean exists(byte[] senderPublicKey, String recipient, byte[] messageData) throws DataException; + +} diff --git a/src/main/java/org/qortal/repository/Repository.java b/src/main/java/org/qortal/repository/Repository.java index 5c28253b7..c0bdb0d94 100644 --- a/src/main/java/org/qortal/repository/Repository.java +++ b/src/main/java/org/qortal/repository/Repository.java @@ -1,5 +1,8 @@ package org.qortal.repository; +import java.io.IOException; +import java.util.concurrent.TimeoutException; + public interface Repository extends AutoCloseable { public ATRepository getATRepository(); @@ -12,10 +15,16 @@ public interface Repository extends AutoCloseable { public BlockRepository getBlockRepository(); + public BlockArchiveRepository getBlockArchiveRepository(); + public ChatRepository getChatRepository(); + public CrossChainRepository getCrossChainRepository(); + public GroupRepository getGroupRepository(); + public MessageRepository getMessageRepository(); + public NameRepository getNameRepository(); public NetworkRepository getNetworkRepository(); @@ -41,6 +50,16 @@ public interface Repository extends AutoCloseable { public void setDebug(boolean debugState); - public void backup(boolean quick) throws DataException; + public void backup(boolean quick, String name, Long timeout) throws DataException, TimeoutException; + + public void performPeriodicMaintenance(Long timeout) throws DataException, TimeoutException; + + public void exportNodeLocalData() throws DataException; + + public void importDataFromFile(String filename) throws DataException, IOException; + + public void checkConsistency() throws DataException; + + public static void attemptRecovery(String connectionUrl, String name) throws DataException {} } diff --git a/src/main/java/org/qortal/repository/RepositoryFactory.java b/src/main/java/org/qortal/repository/RepositoryFactory.java index 94c9627eb..bb34d1c91 100644 --- a/src/main/java/org/qortal/repository/RepositoryFactory.java +++ b/src/main/java/org/qortal/repository/RepositoryFactory.java @@ -1,7 +1,11 @@ package org.qortal.repository; +import java.sql.SQLException; + public interface RepositoryFactory { + public boolean wasPristineAtOpen(); + public RepositoryFactory reopen() throws DataException; public Repository getRepository() throws DataException; @@ -10,4 +14,7 @@ public interface RepositoryFactory { public void close() throws DataException; + // Not ideal place for this but implementating class will know the answer without having to open a new DB session + public boolean isDeadlockException(SQLException e); + } diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index e375be961..714ada281 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -1,13 +1,39 @@ package org.qortal.repository; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.gui.SplashFrame; +import org.qortal.repository.hsqldb.HSQLDBDatabaseArchiving; +import org.qortal.repository.hsqldb.HSQLDBDatabasePruning; +import org.qortal.repository.hsqldb.HSQLDBRepository; +import org.qortal.settings.Settings; + +import java.sql.SQLException; +import java.util.concurrent.TimeoutException; + public abstract class RepositoryManager { + private static final Logger LOGGER = LogManager.getLogger(RepositoryManager.class); private static RepositoryFactory repositoryFactory = null; + /** null if no checkpoint requested, TRUE for quick checkpoint, false for slow/full checkpoint. */ + private static Boolean quickCheckpointRequested = null; + + public static RepositoryFactory getRepositoryFactory() { + return repositoryFactory; + } + public static void setRepositoryFactory(RepositoryFactory newRepositoryFactory) { repositoryFactory = newRepositoryFactory; } + public static boolean wasPristineAtOpen() throws DataException { + if (repositoryFactory == null) + throw new DataException("No repository available"); + + return repositoryFactory.wasPristineAtOpen(); + } + public static Repository getRepository() throws DataException { if (repositoryFactory == null) throw new DataException("No repository available"); @@ -27,14 +53,68 @@ public static void closeRepositoryFactory() throws DataException { repositoryFactory = null; } - public static void backup(boolean quick) { + public static void backup(boolean quick, String name, Long timeout) throws TimeoutException { try (final Repository repository = getRepository()) { - repository.backup(quick); + repository.backup(quick, name, timeout); } catch (DataException e) { // Backup is best-effort so don't complain } } + public static boolean archive(Repository repository) { + // Bulk archive the database the first time we use archive mode + if (Settings.getInstance().isArchiveEnabled()) { + if (RepositoryManager.canArchiveOrPrune()) { + try { + return HSQLDBDatabaseArchiving.buildBlockArchive(repository, BlockArchiveWriter.DEFAULT_FILE_SIZE_TARGET); + + } catch (DataException e) { + LOGGER.info("Unable to build block archive. The database may have been left in an inconsistent state."); + } + } + else { + LOGGER.info("Unable to build block archive due to missing ATStatesHeightIndex. Bootstrapping is recommended."); + LOGGER.info("To bootstrap, stop the core and delete the db folder, then start the core again."); + SplashFrame.getInstance().updateStatus("Missing index. Bootstrapping is recommended."); + } + } + return false; + } + + public static boolean prune(Repository repository) { + // Bulk prune the database the first time we use top-only or block archive mode + if (Settings.getInstance().isTopOnly() || + Settings.getInstance().isArchiveEnabled()) { + if (RepositoryManager.canArchiveOrPrune()) { + try { + boolean prunedATStates = HSQLDBDatabasePruning.pruneATStates((HSQLDBRepository) repository); + boolean prunedBlocks = HSQLDBDatabasePruning.pruneBlocks((HSQLDBRepository) repository); + + // Perform repository maintenance to shrink the db size down + if (prunedATStates && prunedBlocks) { + HSQLDBDatabasePruning.performMaintenance(repository); + return true; + } + + } catch (SQLException | DataException e) { + LOGGER.info("Unable to bulk prune AT states. The database may have been left in an inconsistent state."); + } + } + else { + LOGGER.info("Unable to prune blocks due to missing ATStatesHeightIndex. Bootstrapping is recommended."); + } + } + return false; + } + + public static void setRequestedCheckpoint(Boolean quick) { + quickCheckpointRequested = quick; + } + + public static Boolean getRequestedCheckpoint() { + return quickCheckpointRequested; + } + public static void rebuild() throws DataException { RepositoryFactory oldRepositoryFactory = repositoryFactory; @@ -47,4 +127,18 @@ public static void rebuild() throws DataException { repositoryFactory = oldRepositoryFactory.reopen(); } + public static boolean isDeadlockRelated(Throwable e) { + Throwable cause = e.getCause(); + + return SQLException.class.isInstance(cause) && repositoryFactory.isDeadlockException((SQLException) cause); + } + + public static boolean canArchiveOrPrune() { + try (final Repository repository = getRepository()) { + return repository.getATRepository().hasAtStatesHeightIndex(); + } catch (DataException e) { + return false; + } + } + } diff --git a/src/main/java/org/qortal/repository/TransactionRepository.java b/src/main/java/org/qortal/repository/TransactionRepository.java index 56e51be14..20096eb80 100644 --- a/src/main/java/org/qortal/repository/TransactionRepository.java +++ b/src/main/java/org/qortal/repository/TransactionRepository.java @@ -1,9 +1,11 @@ package org.qortal.repository; +import java.util.EnumSet; import java.util.List; import java.util.Map; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; +import org.qortal.arbitrary.misc.Service; import org.qortal.data.group.GroupApprovalData; import org.qortal.data.transaction.GroupApprovalTransactionData; import org.qortal.data.transaction.TransactionData; @@ -69,8 +71,8 @@ public interface TransactionRepository { * @throws DataException */ public List getSignaturesMatchingCriteria(Integer startBlock, Integer blockLimit, Integer txGroupId, - List txTypes, Integer service, String address, - ConfirmationStatus confirmationStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; + List txTypes, Service service, String name, String address, + ConfirmationStatus confirmationStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; /** * Returns signatures for transactions that match search criteria. @@ -90,6 +92,39 @@ public List getSignaturesMatchingCriteria(Integer startBlock, Integer bl public List getSignaturesMatchingCriteria(TransactionType txType, byte[] publicKey, ConfirmationStatus confirmationStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; + /** + * Returns signatures for transactions that match search criteria. + *

+ * Simpler version that only checks accepts one (optional) transaction type, + * and one (optional) public key, within an block height range. + * + * @param txType + * @param publicKey + * @param minBlockHeight + * @param maxBlockHeight + * @return + * @throws DataException + */ + public List getSignaturesMatchingCriteria(TransactionType txType, byte[] publicKey, + Integer minBlockHeight, Integer maxBlockHeight) throws DataException; + + /** + * Returns signatures for transactions that match search criteria. + *

+ * Alternate version that allows for custom where clauses and bind params. + * Only use for very specific use cases, such as the names integrity check. + * Not advised to be used otherwise, given that it could be possible for + * unsanitized inputs to be passed in if not careful. + * + * @param txType + * @param whereClauses + * @param bindParams + * @return + * @throws DataException + */ + public List getSignaturesMatchingCustomCriteria(TransactionType txType, List whereClauses, + List bindParams) throws DataException; + /** * Returns signature for latest auto-update transaction. *

@@ -107,6 +142,17 @@ public List getSignaturesMatchingCriteria(TransactionType txType, byte[] */ public byte[] getLatestAutoUpdateTransaction(TransactionType txType, int txGroupId, Integer service) throws DataException; + /** + * Returns signatures for all name-registration related transactions relating to supplied name. + * Note: this does not currently include ARBITRARY data relating to the name. + * + * @param name + * @param confirmationStatus + * @return + * @throws DataException + */ + public List getTransactionsInvolvingName(String name, ConfirmationStatus confirmationStatus) throws DataException; + /** * Returns list of transactions relating to specific asset ID. * @@ -223,6 +269,26 @@ public default List getUnconfirmedTransactions() throws DataExc return getUnconfirmedTransactions(null, null, null); } + /** + * Returns list of unconfirmed transactions with specified type and/or creator. + *

+ * At least one of txType or creatorPublicKey must be non-null. + * + * @param txType optional + * @param creatorPublicKey optional + * @return list of transactions, or empty if none. + * @throws DataException + */ + public List getUnconfirmedTransactions(TransactionType txType, byte[] creatorPublicKey) throws DataException; + + /** + * Returns list of unconfirmed transactions excluding specified type(s). + * + * @return list of transactions, or empty if none. + * @throws DataException + */ + public List getUnconfirmedTransactions(EnumSet excludedTxTypes) throws DataException; + /** * Remove transaction from unconfirmed transactions pile. * diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index f6de4fb4d..048239253 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -4,14 +4,23 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Set; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.repository.ATRepository; import org.qortal.repository.DataException; +import org.qortal.utils.ByteArray; + +import com.google.common.primitives.Longs; public class HSQLDBATRepository implements ATRepository { + private static final Logger LOGGER = LogManager.getLogger(HSQLDBATRepository.class); + protected HSQLDBRepository repository; public HSQLDBATRepository(HSQLDBRepository repository) { @@ -24,7 +33,7 @@ public HSQLDBATRepository(HSQLDBRepository repository) { public ATData fromATAddress(String atAddress) throws DataException { String sql = "SELECT creator, created_when, version, asset_id, code_bytes, code_hash, " + "is_sleeping, sleep_until_height, is_finished, had_fatal_error, " - + "is_frozen, frozen_balance " + + "is_frozen, frozen_balance, sleep_until_message_timestamp " + "FROM ATs " + "WHERE AT_address = ? LIMIT 1"; @@ -48,12 +57,17 @@ public ATData fromATAddress(String atAddress) throws DataException { boolean hadFatalError = resultSet.getBoolean(10); boolean isFrozen = resultSet.getBoolean(11); - Long frozenBalance = resultSet.getLong(11); + Long frozenBalance = resultSet.getLong(12); if (frozenBalance == 0 && resultSet.wasNull()) frozenBalance = null; + Long sleepUntilMessageTimestamp = resultSet.getLong(13); + if (sleepUntilMessageTimestamp == 0 && resultSet.wasNull()) + sleepUntilMessageTimestamp = null; + return new ATData(atAddress, creatorPublicKey, created, version, assetId, codeBytes, codeHash, - isSleeping, sleepUntilHeight, isFinished, hadFatalError, isFrozen, frozenBalance); + isSleeping, sleepUntilHeight, isFinished, hadFatalError, isFrozen, frozenBalance, + sleepUntilMessageTimestamp); } catch (SQLException e) { throw new DataException("Unable to fetch AT from repository", e); } @@ -68,11 +82,25 @@ public boolean exists(String atAddress) throws DataException { } } + @Override + public byte[] getCreatorPublicKey(String atAddress) throws DataException { + String sql = "SELECT creator FROM ATs WHERE AT_address = ?"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, atAddress)) { + if (resultSet == null) + return null; + + return resultSet.getBytes(1); + } catch (SQLException e) { + throw new DataException("Unable to fetch AT creator's public key from repository", e); + } + } + @Override public List getAllExecutableATs() throws DataException { String sql = "SELECT AT_address, creator, created_when, version, asset_id, code_bytes, code_hash, " + "is_sleeping, sleep_until_height, had_fatal_error, " - + "is_frozen, frozen_balance " + + "is_frozen, frozen_balance, sleep_until_message_timestamp " + "FROM ATs " + "WHERE is_finished = false " + "ORDER BY created_when ASC"; @@ -102,12 +130,17 @@ public List getAllExecutableATs() throws DataException { boolean hadFatalError = resultSet.getBoolean(10); boolean isFrozen = resultSet.getBoolean(11); - Long frozenBalance = resultSet.getLong(11); + Long frozenBalance = resultSet.getLong(12); if (frozenBalance == 0 && resultSet.wasNull()) frozenBalance = null; + Long sleepUntilMessageTimestamp = resultSet.getLong(13); + if (sleepUntilMessageTimestamp == 0 && resultSet.wasNull()) + sleepUntilMessageTimestamp = null; + ATData atData = new ATData(atAddress, creatorPublicKey, created, version, assetId, codeBytes, codeHash, - isSleeping, sleepUntilHeight, isFinished, hadFatalError, isFrozen, frozenBalance); + isSleeping, sleepUntilHeight, isFinished, hadFatalError, isFrozen, frozenBalance, + sleepUntilMessageTimestamp); executableATs.add(atData); } while (resultSet.next()); @@ -121,16 +154,21 @@ public List getAllExecutableATs() throws DataException { @Override public List getATsByFunctionality(byte[] codeHash, Boolean isExecutable, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(512); + List bindParams = new ArrayList<>(); + sql.append("SELECT AT_address, creator, created_when, version, asset_id, code_bytes, ") .append("is_sleeping, sleep_until_height, is_finished, had_fatal_error, ") - .append("is_frozen, frozen_balance ") + .append("is_frozen, frozen_balance, sleep_until_message_timestamp ") .append("FROM ATs ") .append("WHERE code_hash = ? "); + bindParams.add(codeHash); - if (isExecutable != null) - sql.append("AND is_finished = ").append(isExecutable ? "false" : "true"); + if (isExecutable != null) { + sql.append("AND is_finished != ? "); + bindParams.add(isExecutable); + } - sql.append(" ORDER BY created_when "); + sql.append("ORDER BY created_when "); if (reverse != null && reverse) sql.append("DESC"); @@ -138,7 +176,7 @@ public List getATsByFunctionality(byte[] codeHash, Boolean isExecutable, List matchingATs = new ArrayList<>(); - try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), codeHash)) { + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { if (resultSet == null) return matchingATs; @@ -164,8 +202,88 @@ public List getATsByFunctionality(byte[] codeHash, Boolean isExecutable, if (frozenBalance == 0 && resultSet.wasNull()) frozenBalance = null; + Long sleepUntilMessageTimestamp = resultSet.getLong(13); + if (sleepUntilMessageTimestamp == 0 && resultSet.wasNull()) + sleepUntilMessageTimestamp = null; + ATData atData = new ATData(atAddress, creatorPublicKey, created, version, assetId, codeBytes, codeHash, - isSleeping, sleepUntilHeight, isFinished, hadFatalError, isFrozen, frozenBalance); + isSleeping, sleepUntilHeight, isFinished, hadFatalError, isFrozen, frozenBalance, + sleepUntilMessageTimestamp); + + matchingATs.add(atData); + } while (resultSet.next()); + + return matchingATs; + } catch (SQLException e) { + throw new DataException("Unable to fetch matching ATs from repository", e); + } + } + + @Override + public List getAllATsByFunctionality(Set codeHashes, Boolean isExecutable) throws DataException { + StringBuilder sql = new StringBuilder(512); + List bindParams = new ArrayList<>(); + + sql.append("SELECT AT_address, creator, created_when, version, asset_id, code_bytes, ") + .append("is_sleeping, sleep_until_height, is_finished, had_fatal_error, ") + .append("is_frozen, frozen_balance, code_hash, sleep_until_message_timestamp ") + .append("FROM "); + + // (VALUES (?), (?), ...) AS ATCodeHashes (code_hash) + sql.append("(VALUES "); + + boolean isFirst = true; + for (ByteArray codeHash : codeHashes) { + if (!isFirst) + sql.append(", "); + else + isFirst = false; + + sql.append("(CAST(? AS VARBINARY(256)))"); + bindParams.add(codeHash.value); + } + sql.append(") AS ATCodeHashes (code_hash) "); + + sql.append("JOIN ATs ON ATs.code_hash = ATCodeHashes.code_hash "); + + if (isExecutable != null) { + sql.append("AND is_finished != ? "); + bindParams.add(isExecutable); + } + + List matchingATs = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return matchingATs; + + do { + String atAddress = resultSet.getString(1); + byte[] creatorPublicKey = resultSet.getBytes(2); + long created = resultSet.getLong(3); + int version = resultSet.getInt(4); + long assetId = resultSet.getLong(5); + byte[] codeBytes = resultSet.getBytes(6); // Actually BLOB + boolean isSleeping = resultSet.getBoolean(7); + + Integer sleepUntilHeight = resultSet.getInt(8); + if (sleepUntilHeight == 0 && resultSet.wasNull()) + sleepUntilHeight = null; + + boolean isFinished = resultSet.getBoolean(9); + + boolean hadFatalError = resultSet.getBoolean(10); + boolean isFrozen = resultSet.getBoolean(11); + + Long frozenBalance = resultSet.getLong(12); + if (frozenBalance == 0 && resultSet.wasNull()) + frozenBalance = null; + + byte[] codeHash = resultSet.getBytes(13); + Long sleepUntilMessageTimestamp = resultSet.getLong(14); + + ATData atData = new ATData(atAddress, creatorPublicKey, created, version, assetId, codeBytes, codeHash, + isSleeping, sleepUntilHeight, isFinished, hadFatalError, isFrozen, frozenBalance, sleepUntilMessageTimestamp); matchingATs.add(atData); } while (resultSet.next()); @@ -204,7 +322,7 @@ public void save(ATData atData) throws DataException { .bind("code_bytes", atData.getCodeBytes()).bind("code_hash", atData.getCodeHash()) .bind("is_sleeping", atData.getIsSleeping()).bind("sleep_until_height", atData.getSleepUntilHeight()) .bind("is_finished", atData.getIsFinished()).bind("had_fatal_error", atData.getHadFatalError()).bind("is_frozen", atData.getIsFrozen()) - .bind("frozen_balance", atData.getFrozenBalance()); + .bind("frozen_balance", atData.getFrozenBalance()).bind("sleep_until_message_timestamp", atData.getSleepUntilMessageTimestamp()); try { saveHelper.execute(this.repository); @@ -227,22 +345,26 @@ public void delete(String atAddress) throws DataException { @Override public ATStateData getATStateAtHeight(String atAddress, int height) throws DataException { - String sql = "SELECT created_when, state_data, state_hash, fees, is_initial " + String sql = "SELECT state_data, state_hash, fees, is_initial, sleep_until_message_timestamp " + "FROM ATStates " - + "WHERE AT_address = ? AND height = ? " + + "LEFT OUTER JOIN ATStatesData USING (AT_address, height) " + + "WHERE ATStates.AT_address = ? AND ATStates.height = ? " + "LIMIT 1"; try (ResultSet resultSet = this.repository.checkedExecute(sql, atAddress, height)) { if (resultSet == null) return null; - long created = resultSet.getLong(1); - byte[] stateData = resultSet.getBytes(2); // Actually BLOB - byte[] stateHash = resultSet.getBytes(3); - long fees = resultSet.getLong(4); - boolean isInitial = resultSet.getBoolean(5); + byte[] stateData = resultSet.getBytes(1); // Actually BLOB + byte[] stateHash = resultSet.getBytes(2); + long fees = resultSet.getLong(3); + boolean isInitial = resultSet.getBoolean(4); + + Long sleepUntilMessageTimestamp = resultSet.getLong(5); + if (sleepUntilMessageTimestamp == 0 && resultSet.wasNull()) + sleepUntilMessageTimestamp = null; - return new ATStateData(atAddress, height, created, stateData, stateHash, fees, isInitial); + return new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial, sleepUntilMessageTimestamp); } catch (SQLException e) { throw new DataException("Unable to fetch AT state from repository", e); } @@ -250,33 +372,209 @@ public ATStateData getATStateAtHeight(String atAddress, int height) throws DataE @Override public ATStateData getLatestATState(String atAddress) throws DataException { - String sql = "SELECT height, created_when, state_data, state_hash, fees, is_initial " + String sql = "SELECT height, state_data, state_hash, fees, is_initial, sleep_until_message_timestamp " + "FROM ATStates " - + "WHERE AT_address = ? " - + "ORDER BY height DESC " - + "LIMIT 1"; + + "JOIN ATStatesData USING (AT_address, height) " + + "WHERE ATStates.AT_address = ? " + // Order by AT_address and height to use compound primary key as index + // Both must be the same direction (DESC) also + + "ORDER BY ATStates.AT_address DESC, ATStates.height DESC " + + "LIMIT 1 "; try (ResultSet resultSet = this.repository.checkedExecute(sql, atAddress)) { if (resultSet == null) return null; int height = resultSet.getInt(1); - long created = resultSet.getLong(2); - byte[] stateData = resultSet.getBytes(3); // Actually BLOB - byte[] stateHash = resultSet.getBytes(4); - long fees = resultSet.getLong(5); - boolean isInitial = resultSet.getBoolean(6); + byte[] stateData = resultSet.getBytes(2); // Actually BLOB + byte[] stateHash = resultSet.getBytes(3); + long fees = resultSet.getLong(4); + boolean isInitial = resultSet.getBoolean(5); - return new ATStateData(atAddress, height, created, stateData, stateHash, fees, isInitial); + Long sleepUntilMessageTimestamp = resultSet.getLong(6); + if (sleepUntilMessageTimestamp == 0 && resultSet.wasNull()) + sleepUntilMessageTimestamp = null; + + return new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial, sleepUntilMessageTimestamp); } catch (SQLException e) { throw new DataException("Unable to fetch latest AT state from repository", e); } } + @Override + public List getMatchingFinalATStates(byte[] codeHash, Boolean isFinished, + Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight, + Integer limit, Integer offset, Boolean reverse) throws DataException { + StringBuilder sql = new StringBuilder(1024); + List bindParams = new ArrayList<>(); + + sql.append("SELECT AT_address, height, state_data, state_hash, fees, is_initial, FinalATStates.sleep_until_message_timestamp " + + "FROM ATs " + + "CROSS JOIN LATERAL(" + + "SELECT height, state_data, state_hash, fees, is_initial, sleep_until_message_timestamp " + + "FROM ATStates " + + "JOIN ATStatesData USING (AT_address, height) " + + "WHERE ATStates.AT_address = ATs.AT_address "); + + if (minimumFinalHeight != null) { + sql.append("AND ATStates.height >= ? "); + bindParams.add(minimumFinalHeight); + } + + // Order by AT_address and height to use compound primary key as index + // Both must be the same direction (DESC) also + sql.append("ORDER BY ATStates.AT_address DESC, ATStates.height DESC " + + "LIMIT 1 " + + ") AS FinalATStates " + + "WHERE code_hash = ? "); + bindParams.add(codeHash); + + if (isFinished != null) { + sql.append("AND is_finished = ? "); + bindParams.add(isFinished); + } + + if (dataByteOffset != null && expectedValue != null) { + sql.append("AND SUBSTRING(state_data FROM ? FOR 8) = ? "); + + // We convert our long on Java-side to control endian + byte[] rawExpectedValue = Longs.toByteArray(expectedValue); + + // SQL binary data offsets start at 1 + bindParams.add(dataByteOffset + 1); + bindParams.add(rawExpectedValue); + } + + sql.append(" ORDER BY FinalATStates.height "); + if (reverse != null && reverse) + sql.append("DESC"); + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List atStates = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return atStates; + + do { + String atAddress = resultSet.getString(1); + int height = resultSet.getInt(2); + byte[] stateData = resultSet.getBytes(3); // Actually BLOB + byte[] stateHash = resultSet.getBytes(4); + long fees = resultSet.getLong(5); + boolean isInitial = resultSet.getBoolean(6); + + Long sleepUntilMessageTimestamp = resultSet.getLong(7); + if (sleepUntilMessageTimestamp == 0 && resultSet.wasNull()) + sleepUntilMessageTimestamp = null; + + ATStateData atStateData = new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial, sleepUntilMessageTimestamp); + + atStates.add(atStateData); + } while (resultSet.next()); + + return atStates; + } catch (SQLException e) { + throw new DataException("Unable to fetch matching AT states from repository", e); + } + } + + @Override + public List getMatchingFinalATStatesQuorum(byte[] codeHash, Boolean isFinished, + Integer dataByteOffset, Long expectedValue, + int minimumCount, int maximumCount, long minimumPeriod) throws DataException { + // We need most recent entry first so we can use its timestamp to slice further results + List mostRecentStates = this.getMatchingFinalATStates(codeHash, isFinished, + dataByteOffset, expectedValue, null, + 1, 0, true); + + if (mostRecentStates == null) + return null; + + if (mostRecentStates.isEmpty()) + return mostRecentStates; + + ATStateData mostRecentState = mostRecentStates.get(0); + + StringBuilder sql = new StringBuilder(1024); + List bindParams = new ArrayList<>(); + + sql.append("SELECT AT_address, height, state_data, state_hash, fees, is_initial, sleep_until_message_timestamp " + + "FROM ATs " + + "CROSS JOIN LATERAL(" + + "SELECT height, state_data, state_hash, fees, is_initial " + + "FROM ATStates " + + "JOIN ATStatesData USING (AT_address, height) " + + "WHERE ATStates.AT_address = ATs.AT_address "); + + // Order by AT_address and height to use compound primary key as index + // Both must be the same direction (DESC) also + sql.append("ORDER BY ATStates.AT_address DESC, ATStates.height DESC " + + "LIMIT 1 " + + ") AS FinalATStates " + + "WHERE code_hash = ? "); + bindParams.add(codeHash); + + if (isFinished != null) { + sql.append("AND is_finished = ? "); + bindParams.add(isFinished); + } + + if (dataByteOffset != null && expectedValue != null) { + sql.append("AND SUBSTRING(state_data FROM ? FOR 8) = ? "); + + // We convert our long on Java-side to control endian + byte[] rawExpectedValue = Longs.toByteArray(expectedValue); + + // SQL binary data offsets start at 1 + bindParams.add(dataByteOffset + 1); + bindParams.add(rawExpectedValue); + } + + // Slice so that we meet both minimumCount and minimumPeriod + int minimumHeight = mostRecentState.getHeight() - (int) (minimumPeriod / 60 * 1000L); // XXX assumes 60 second blocks + + sql.append("AND (FinalATStates.height >= ? OR ROWNUM() < ?) "); + bindParams.add(minimumHeight); + bindParams.add(minimumCount); + + sql.append("ORDER BY FinalATStates.height DESC LIMIT ?"); + bindParams.add(maximumCount); + + List atStates = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return atStates; + + do { + String atAddress = resultSet.getString(1); + int height = resultSet.getInt(2); + byte[] stateData = resultSet.getBytes(3); // Actually BLOB + byte[] stateHash = resultSet.getBytes(4); + long fees = resultSet.getLong(5); + boolean isInitial = resultSet.getBoolean(6); + Long sleepUntilMessageTimestamp = resultSet.getLong(7); + + ATStateData atStateData = new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial, + sleepUntilMessageTimestamp); + + atStates.add(atStateData); + } while (resultSet.next()); + + return atStates; + } catch (SQLException e) { + throw new DataException("Unable to fetch matching AT states from repository", e); + } + } + @Override public List getBlockATStatesAtHeight(int height) throws DataException { String sql = "SELECT AT_address, state_hash, fees, is_initial " - + "FROM ATStates " + + "FROM ATs " + + "JOIN ATStates " + + "ON ATStates.AT_address = ATs.AT_address " + "WHERE height = ? " + "ORDER BY created_when ASC"; @@ -303,30 +601,261 @@ public List getBlockATStatesAtHeight(int height) throws DataExcepti return atStates; } + + @Override + public void rebuildLatestAtStates() throws DataException { + // latestATStatesLock is to prevent concurrent updates on LatestATStates + // that could result in one process using a partial or empty dataset + // because it was in the process of being rebuilt by another thread + synchronized (this.repository.latestATStatesLock) { + LOGGER.trace("Rebuilding latest AT states..."); + + // Rebuild cache of latest AT states that we can't trim + String deleteSql = "DELETE FROM LatestATStates"; + try { + this.repository.executeCheckedUpdate(deleteSql); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to delete temporary latest AT states cache from repository", e); + } + + String insertSql = "INSERT INTO LatestATStates (" + + "SELECT AT_address, height FROM ATs " + + "CROSS JOIN LATERAL(" + + "SELECT height FROM ATStates " + + "WHERE ATStates.AT_address = ATs.AT_address " + + "ORDER BY AT_address DESC, height DESC LIMIT 1" + + ") " + + ")"; + try { + this.repository.executeCheckedUpdate(insertSql); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to populate temporary latest AT states cache in repository", e); + } + this.repository.saveChanges(); + LOGGER.trace("Rebuilt latest AT states"); + } + } + + + @Override + public int getAtTrimHeight() throws DataException { + String sql = "SELECT AT_trim_height FROM DatabaseInfo"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return 0; + + return resultSet.getInt(1); + } catch (SQLException e) { + throw new DataException("Unable to fetch AT state trim height from repository", e); + } + } + + @Override + public void setAtTrimHeight(int trimHeight) throws DataException { + // trimHeightsLock is to prevent concurrent update on DatabaseInfo + // that could result in "transaction rollback: serialization failure" + synchronized (this.repository.trimHeightsLock) { + String updateSql = "UPDATE DatabaseInfo SET AT_trim_height = ?"; + + try { + this.repository.executeCheckedUpdate(updateSql, trimHeight); + this.repository.saveChanges(); + } catch (SQLException e) { + this.repository.examineException(e); + throw new DataException("Unable to set AT state trim height in repository", e); + } + } + } + + @Override + public int trimAtStates(int minHeight, int maxHeight, int limit) throws DataException { + if (minHeight >= maxHeight) + return 0; + + // latestATStatesLock is to prevent concurrent updates on LatestATStates + // that could result in one process using a partial or empty dataset + // because it was in the process of being rebuilt by another thread + synchronized (this.repository.latestATStatesLock) { + + // We're often called so no need to trim all states in one go. + // Limit updates to reduce CPU and memory load. + String sql = "DELETE FROM ATStatesData " + + "WHERE height BETWEEN ? AND ? " + + "AND NOT EXISTS(" + + "SELECT TRUE FROM LatestATStates " + + "WHERE LatestATStates.AT_address = ATStatesData.AT_address " + + "AND LatestATStates.height = ATStatesData.height" + + ") " + + "LIMIT ?"; + + try { + int modifiedRows = this.repository.executeCheckedUpdate(sql, minHeight, maxHeight, limit); + this.repository.saveChanges(); + return modifiedRows; + + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to trim AT states in repository", e); + } + } + } + + + @Override + public int getAtPruneHeight() throws DataException { + String sql = "SELECT AT_prune_height FROM DatabaseInfo"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return 0; + + return resultSet.getInt(1); + } catch (SQLException e) { + throw new DataException("Unable to fetch AT state prune height from repository", e); + } + } + + @Override + public void setAtPruneHeight(int pruneHeight) throws DataException { + // trimHeightsLock is to prevent concurrent update on DatabaseInfo + // that could result in "transaction rollback: serialization failure" + synchronized (this.repository.trimHeightsLock) { + String updateSql = "UPDATE DatabaseInfo SET AT_prune_height = ?"; + + try { + this.repository.executeCheckedUpdate(updateSql, pruneHeight); + this.repository.saveChanges(); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to set AT state prune height in repository", e); + } + } + } + + @Override + public int pruneAtStates(int minHeight, int maxHeight) throws DataException { + // latestATStatesLock is to prevent concurrent updates on LatestATStates + // that could result in one process using a partial or empty dataset + // because it was in the process of being rebuilt by another thread + synchronized (this.repository.latestATStatesLock) { + + int deletedCount = 0; + + for (int height = minHeight; height <= maxHeight; height++) { + + // Give up if we're stopping + if (Controller.isStopping()) { + return deletedCount; + } + + // Get latest AT states for this height + List atAddresses = new ArrayList<>(); + String updateSql = "SELECT AT_address FROM LatestATStates WHERE height = ?"; + try (ResultSet resultSet = this.repository.checkedExecute(updateSql, height)) { + if (resultSet != null) { + do { + String atAddress = resultSet.getString(1); + atAddresses.add(atAddress); + + } while (resultSet.next()); + } + } catch (SQLException e) { + throw new DataException("Unable to fetch latest AT states from repository", e); + } + + List atStates = this.getBlockATStatesAtHeight(height); + for (ATStateData atState : atStates) { + //LOGGER.info("Found atState {} at height {}", atState.getATAddress(), atState.getHeight()); + + // Give up if we're stopping + if (Controller.isStopping()) { + return deletedCount; + } + + if (atAddresses.contains(atState.getATAddress())) { + // We don't want to delete this AT state because it is still active + LOGGER.trace("Skipping atState {} at height {}", atState.getATAddress(), atState.getHeight()); + continue; + } + + // Safe to delete everything else for this height + try { + this.repository.delete("ATStates", "AT_address = ? AND height = ?", + atState.getATAddress(), atState.getHeight()); + deletedCount++; + } catch (SQLException e) { + throw new DataException("Unable to delete AT state data from repository", e); + } + } + } + this.repository.saveChanges(); + + return deletedCount; + } + } + + + @Override + public boolean hasAtStatesHeightIndex() throws DataException { + String sql = "SELECT INDEX_NAME FROM INFORMATION_SCHEMA.SYSTEM_INDEXINFO where INDEX_NAME='ATSTATESHEIGHTINDEX'"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + return resultSet != null; + + } catch (SQLException e) { + throw new DataException("Unable to check for ATStatesHeightIndex in repository", e); + } + } + + @Override public void save(ATStateData atStateData) throws DataException { // We shouldn't ever save partial ATStateData - if (atStateData.getCreation() == null || atStateData.getStateHash() == null || atStateData.getHeight() == null) + if (atStateData.getStateHash() == null || atStateData.getHeight() == null) throw new IllegalArgumentException("Refusing to save partial AT state into repository!"); - HSQLDBSaver saveHelper = new HSQLDBSaver("ATStates"); + HSQLDBSaver atStatesSaver = new HSQLDBSaver("ATStates"); - saveHelper.bind("AT_address", atStateData.getATAddress()).bind("height", atStateData.getHeight()) - .bind("created_when", atStateData.getCreation()).bind("state_data", atStateData.getStateData()) - .bind("state_hash", atStateData.getStateHash()).bind("fees", atStateData.getFees()) - .bind("is_initial", atStateData.isInitial()); + atStatesSaver.bind("AT_address", atStateData.getATAddress()).bind("height", atStateData.getHeight()) + .bind("state_hash", atStateData.getStateHash()) + .bind("fees", atStateData.getFees()).bind("is_initial", atStateData.isInitial()) + .bind("sleep_until_message_timestamp", atStateData.getSleepUntilMessageTimestamp()); try { - saveHelper.execute(this.repository); + atStatesSaver.execute(this.repository); } catch (SQLException e) { throw new DataException("Unable to save AT state into repository", e); } + + if (atStateData.getStateData() != null) { + HSQLDBSaver atStatesDataSaver = new HSQLDBSaver("ATStatesData"); + + atStatesDataSaver.bind("AT_address", atStateData.getATAddress()).bind("height", atStateData.getHeight()) + .bind("state_data", atStateData.getStateData()); + + try { + atStatesDataSaver.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save AT state data into repository", e); + } + } else { + try { + this.repository.delete("ATStatesData", "AT_address = ? AND height = ?", + atStateData.getATAddress(), atStateData.getHeight()); + } catch (SQLException e) { + throw new DataException("Unable to delete AT state data from repository", e); + } + } } @Override public void delete(String atAddress, int height) throws DataException { try { this.repository.delete("ATStates", "AT_address = ? AND height = ?", atAddress, height); + this.repository.delete("ATStatesData", "AT_address = ? AND height = ?", atAddress, height); } catch (SQLException e) { throw new DataException("Unable to delete AT state from repository", e); } @@ -336,9 +865,74 @@ public void delete(String atAddress, int height) throws DataException { public void deleteATStates(int height) throws DataException { try { this.repository.delete("ATStates", "height = ?", height); + this.repository.delete("ATStatesData", "height = ?", height); } catch (SQLException e) { throw new DataException("Unable to delete AT states from repository", e); } } + // Finding transactions for ATs to process + + public NextTransactionInfo findNextTransaction(String recipient, int height, int sequence) throws DataException { + // We only need to search for a subset of transaction types: MESSAGE, PAYMENT or AT + + String sql = "SELECT height, sequence, Transactions.signature " + + "FROM (" + + "SELECT signature FROM PaymentTransactions WHERE recipient = ? " + + "UNION " + + "SELECT signature FROM MessageTransactions WHERE recipient = ? " + + "UNION " + + "SELECT signature FROM ATTransactions WHERE recipient = ?" + + ") AS Transactions " + + "JOIN BlockTransactions ON BlockTransactions.transaction_signature = Transactions.signature " + + "JOIN Blocks ON Blocks.signature = BlockTransactions.block_signature " + + "WHERE (height > ? OR (height = ? AND sequence > ?)) " + + "ORDER BY height ASC, sequence ASC " + + "LIMIT 1"; + + Object[] bindParams = new Object[] { recipient, recipient, recipient, height, height, sequence }; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, bindParams)) { + if (resultSet == null) + return null; + + int nextHeight = resultSet.getInt(1); + int nextSequence = resultSet.getInt(2); + byte[] nextSignature = resultSet.getBytes(3); + + return new NextTransactionInfo(nextHeight, nextSequence, nextSignature); + } catch (SQLException e) { + throw new DataException("Unable to find next transaction to AT from repository", e); + } + } + + // Other + + public void checkConsistency() throws DataException { + String sql = "SELECT COUNT(*) FROM ATs " + + "CROSS JOIN LATERAL(" + + "SELECT height FROM ATStates " + + "WHERE ATStates.AT_address = ATs.AT_address " + + "ORDER BY AT_address DESC, height DESC " + + "LIMIT 1" + + ") AS LatestATState (height) " + + "LEFT OUTER JOIN ATStatesData " + + "ON ATStatesData.AT_address = ATs.AT_address AND ATStatesData.height = LatestATState.height " + + "WHERE ATStatesData.AT_address IS NULL"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + throw new DataException("Unable to check AT repository consistency"); + + int atCount = resultSet.getInt(1); + + if (atCount > 0) { + LOGGER.warn(() -> String.format("Missing %d latest AT state data row%s!", atCount, (atCount != 1 ? "s" : ""))); + LOGGER.warn("Export key data then resync using bootstrap as soon as possible"); + } + } catch (SQLException e) { + throw new DataException("Unable to check AT repository consistency", e); + } + } + } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java index 928a251b7..b28a224c0 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java @@ -1,13 +1,17 @@ package org.qortal.repository.hsqldb; +import static org.qortal.utils.Amounts.prettyAmount; + import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import org.qortal.asset.Asset; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; +import org.qortal.data.account.EligibleQoraHolderData; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.QortFromQoraData; import org.qortal.data.account.RewardShareData; @@ -145,7 +149,7 @@ public boolean accountExists(String address) throws DataException { public void ensureAccount(AccountData accountData) throws DataException { String sql = "INSERT IGNORE INTO Accounts (account, public_key) VALUES (?, ?)"; // MySQL syntax try { - this.repository.checkedExecuteUpdateCount(sql, accountData.getAddress(), accountData.getPublicKey()); + this.repository.executeCheckedUpdate(sql, accountData.getAddress(), accountData.getPublicKey()); } catch (SQLException e) { throw new DataException("Unable to ensure minimal account in repository", e); } @@ -260,12 +264,26 @@ public int modifyMintedBlockCount(String address, int delta) throws DataExceptio "ON DUPLICATE KEY UPDATE blocks_minted = blocks_minted + ?"; try { - return this.repository.checkedExecuteUpdateCount(sql, address, delta, delta); + return this.repository.executeCheckedUpdate(sql, address, delta, delta); } catch (SQLException e) { throw new DataException("Unable to modify account's minted block count in repository", e); } } + @Override + public void modifyMintedBlockCounts(List addresses, int delta) throws DataException { + String sql = "INSERT INTO Accounts (account, blocks_minted) VALUES (?, ?) " + + "ON DUPLICATE KEY UPDATE blocks_minted = blocks_minted + ?"; + + List bindParamRows = addresses.stream().map(address -> new Object[] { address, delta, delta }).collect(Collectors.toList()); + + try { + this.repository.executeCheckedBatchUpdate(sql, bindParamRows); + } catch (SQLException e) { + throw new DataException("Unable to modify many account minted block counts in repository", e); + } + } + @Override public void delete(String address) throws DataException { // NOTE: Account balances are deleted automatically by the database thanks to "ON DELETE CASCADE" in AccountBalances' FOREIGN KEY @@ -447,7 +465,7 @@ public void modifyAssetBalance(String address, long assetId, long deltaBalance) // Perform actual balance change String sql = "UPDATE AccountBalances set balance = balance + ? WHERE account = ? AND asset_id = ?"; try { - this.repository.checkedExecuteUpdateCount(sql, deltaBalance, address, assetId); + this.repository.executeCheckedUpdate(sql, deltaBalance, address, assetId); } catch (SQLException e) { throw new DataException("Unable to reduce account balance in repository", e); } @@ -455,7 +473,7 @@ public void modifyAssetBalance(String address, long assetId, long deltaBalance) // We have to ensure parent row exists to satisfy foreign key constraint try { String sql = "INSERT IGNORE INTO Accounts (account) VALUES (?)"; // MySQL syntax - this.repository.checkedExecuteUpdateCount(sql, address); + this.repository.executeCheckedUpdate(sql, address); } catch (SQLException e) { throw new DataException("Unable to ensure minimal account in repository", e); } @@ -464,13 +482,95 @@ public void modifyAssetBalance(String address, long assetId, long deltaBalance) String sql = "INSERT INTO AccountBalances (account, asset_id, balance) VALUES (?, ?, ?) " + "ON DUPLICATE KEY UPDATE balance = balance + ?"; try { - this.repository.checkedExecuteUpdateCount(sql, address, assetId, deltaBalance, deltaBalance); + this.repository.executeCheckedUpdate(sql, address, assetId, deltaBalance, deltaBalance); } catch (SQLException e) { throw new DataException("Unable to increase account balance in repository", e); } } } + public void modifyAssetBalances(List accountBalanceDeltas) throws DataException { + // Nothing to do? + if (accountBalanceDeltas == null || accountBalanceDeltas.isEmpty()) + return; + + // Map balance changes into SQL bind params, filtering out no-op changes + List modifyBalanceParams = accountBalanceDeltas.stream() + .filter(accountBalance -> accountBalance.getBalance() != 0L) + .map(accountBalance -> new Object[] { accountBalance.getAddress(), accountBalance.getAssetId(), accountBalance.getBalance(), accountBalance.getBalance() }) + .collect(Collectors.toList()); + + // Before we modify balances, ensure parent accounts exist + String ensureSql = "INSERT IGNORE INTO Accounts (account) VALUES (?)"; // MySQL syntax + try { + this.repository.executeCheckedBatchUpdate(ensureSql, modifyBalanceParams.stream().map(objects -> new Object[] { objects[0] }).collect(Collectors.toList())); + } catch (SQLException e) { + throw new DataException("Unable to ensure minimal accounts in repository", e); + } + + // Perform actual balance changes + String sql = "INSERT INTO AccountBalances (account, asset_id, balance) VALUES (?, ?, ?) " + + "ON DUPLICATE KEY UPDATE balance = balance + ?"; + try { + this.repository.executeCheckedBatchUpdate(sql, modifyBalanceParams); + } catch (SQLException e) { + throw new DataException("Unable to modify account balances in repository", e); + } + } + + + @Override + public void setAssetBalances(List accountBalances) throws DataException { + // Nothing to do? + if (accountBalances == null || accountBalances.isEmpty()) + return; + + /* + * Split workload into zero and non-zero balances, + * checking for negative balances as we progress. + */ + + List zeroAccountBalanceParams = new ArrayList<>(); + List nonZeroAccountBalanceParams = new ArrayList<>(); + + for (AccountBalanceData accountBalanceData : accountBalances) { + final long balance = accountBalanceData.getBalance(); + + if (balance < 0) + throw new DataException(String.format("Refusing to set negative balance %s [assetId %d] for %s", + prettyAmount(balance), accountBalanceData.getAssetId(), accountBalanceData.getAddress())); + + if (balance == 0) + zeroAccountBalanceParams.add(new Object[] { accountBalanceData.getAddress(), accountBalanceData.getAssetId() }); + else + nonZeroAccountBalanceParams.add(new Object[] { accountBalanceData.getAddress(), accountBalanceData.getAssetId(), balance, balance }); + } + + // Batch update (actually delete) of zero balances + try { + this.repository.deleteBatch("AccountBalances", "account = ? AND asset_id = ?", zeroAccountBalanceParams); + } catch (SQLException e) { + throw new DataException("Unable to delete account balances from repository", e); + } + + // Before we set new balances, ensure parent accounts exist + String ensureSql = "INSERT IGNORE INTO Accounts (account) VALUES (?)"; // MySQL syntax + try { + this.repository.executeCheckedBatchUpdate(ensureSql, nonZeroAccountBalanceParams.stream().map(objects -> new Object[] { objects[0] }).collect(Collectors.toList())); + } catch (SQLException e) { + throw new DataException("Unable to ensure minimal accounts in repository", e); + } + + // Now set all balances in one go + String setSql = "INSERT INTO AccountBalances (account, asset_id, balance) VALUES (?, ?, ?) " + + "ON DUPLICATE KEY UPDATE balance = ?"; + try { + this.repository.executeCheckedBatchUpdate(setSql, nonZeroAccountBalanceParams); + } catch (SQLException e) { + throw new DataException("Unable to set account balances in repository", e); + } + } + @Override public void save(AccountBalanceData accountBalanceData) throws DataException { HSQLDBSaver saveHelper = new HSQLDBSaver("AccountBalances"); @@ -699,7 +799,52 @@ public RewardShareData getRewardShareByIndex(int index) throws DataException { return new RewardShareData(minterPublicKey, minter, recipient, rewardSharePublicKey, sharePercent); } catch (SQLException e) { - throw new DataException("Unable to fetch reward-share info from repository", e); + throw new DataException("Unable to fetch indexed reward-share from repository", e); + } + } + + @Override + public List getRewardSharesByIndexes(int[] indexes) throws DataException { + String sql = "SELECT minter_public_key, minter, recipient, share_percent, reward_share_public_key FROM RewardShares " + + "ORDER BY reward_share_public_key ASC"; + + if (indexes == null) + return null; + + List rewardShares = new ArrayList<>(); + if (indexes.length == 0) + return rewardShares; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return null; + + int rowNum = 1; + for (int i = 0; i < indexes.length; ++i) { + final int index = indexes[i]; + + while (rowNum < index + 1) { // +1 because in JDBC, first row is row 1 + if (!resultSet.next()) + // Index is out of bounds + return null; + + ++rowNum; + } + + byte[] minterPublicKey = resultSet.getBytes(1); + String minter = resultSet.getString(2); + String recipient = resultSet.getString(3); + int sharePercent = resultSet.getInt(4); + byte[] rewardSharePublicKey = resultSet.getBytes(5); + + RewardShareData rewardShareData = new RewardShareData(minterPublicKey, minter, recipient, rewardSharePublicKey, sharePercent); + + rewardShares.add(rewardShareData); + } + + return rewardShares; + } catch (SQLException e) { + throw new DataException("Unable to fetch indexed reward-shares from repository", e); } } @@ -759,6 +904,25 @@ public List getMintingAccounts() throws DataException { } } + @Override + public MintingAccountData getMintingAccount(byte[] mintingAccountKey) throws DataException { + try (ResultSet resultSet = this.repository.checkedExecute("SELECT minter_private_key, minter_public_key " + + "FROM MintingAccounts WHERE minter_private_key = ? OR minter_public_key = ?", + mintingAccountKey, mintingAccountKey)) { + + if (resultSet == null) + return null; + + byte[] minterPrivateKey = resultSet.getBytes(1); + byte[] minterPublicKey = resultSet.getBytes(2); + + return new MintingAccountData(minterPrivateKey, minterPublicKey); + + } catch (SQLException e) { + throw new DataException("Unable to fetch minting accounts from repository", e); + } + } + @Override public void save(MintingAccountData mintingAccountData) throws DataException { HSQLDBSaver saveHelper = new HSQLDBSaver("MintingAccounts"); @@ -774,9 +938,9 @@ public void save(MintingAccountData mintingAccountData) throws DataException { } @Override - public int delete(byte[] minterPrivateKey) throws DataException { + public int delete(byte[] minterKey) throws DataException { try { - return this.repository.delete("MintingAccounts", "minter_private_key = ?", minterPrivateKey); + return this.repository.delete("MintingAccounts", "minter_private_key = ? OR minter_public_key = ?", minterKey, minterKey); } catch (SQLException e) { throw new DataException("Unable to delete minting account from repository", e); } @@ -785,35 +949,49 @@ public int delete(byte[] minterPrivateKey) throws DataException { // Managing QORT from legacy QORA @Override - public List getEligibleLegacyQoraHolders(Integer blockHeight) throws DataException { + public List getEligibleLegacyQoraHolders(Integer blockHeight) throws DataException { StringBuilder sql = new StringBuilder(1024); - sql.append("SELECT account, balance from AccountBalances "); + List bindParams = new ArrayList<>(); + + sql.append("SELECT account, Qora.balance, QortFromQora.balance, final_qort_from_qora, final_block_height "); + sql.append("FROM AccountBalances AS Qora "); sql.append("LEFT OUTER JOIN AccountQortFromQoraInfo USING (account) "); - sql.append("WHERE asset_id = "); + sql.append("LEFT OUTER JOIN AccountBalances AS QortFromQora ON QortFromQora.account = Qora.account AND QortFromQora.asset_id = "); + sql.append(Asset.QORT_FROM_QORA); // int is safe to use literally + sql.append(" WHERE Qora.asset_id = "); sql.append(Asset.LEGACY_QORA); // int is safe to use literally sql.append(" AND (final_block_height IS NULL"); if (blockHeight != null) { - sql.append(" OR final_block_height >= "); - sql.append(blockHeight); + sql.append(" OR final_block_height >= ?"); + bindParams.add(blockHeight); } sql.append(")"); - List accountBalances = new ArrayList<>(); + List eligibleLegacyQoraHolders = new ArrayList<>(); - try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) { + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { if (resultSet == null) - return accountBalances; + return eligibleLegacyQoraHolders; do { String address = resultSet.getString(1); - long balance = resultSet.getLong(2); + long qoraBalance = resultSet.getLong(2); + long qortFromQoraBalance = resultSet.getLong(3); - accountBalances.add(new AccountBalanceData(address, Asset.LEGACY_QORA, balance)); + Long finalQortFromQora = resultSet.getLong(4); + if (finalQortFromQora == 0 && resultSet.wasNull()) + finalQortFromQora = null; + + Integer finalBlockHeight = resultSet.getInt(5); + if (finalBlockHeight == 0 && resultSet.wasNull()) + finalBlockHeight = null; + + eligibleLegacyQoraHolders.add(new EligibleQoraHolderData(address, qoraBalance, qortFromQoraBalance, finalQortFromQora, finalBlockHeight)); } while (resultSet.next()); - return accountBalances; + return eligibleLegacyQoraHolders; } catch (SQLException e) { throw new DataException("Unable to fetch eligible legacy QORA holders from repository", e); } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 3d99bbb32..a7da66ae9 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -1,58 +1,40 @@ package org.qortal.repository.hsqldb; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bouncycastle.util.Longs; +import org.qortal.arbitrary.misc.Service; +import org.qortal.data.arbitrary.ArbitraryResourceInfo; import org.qortal.crypto.Crypto; +import org.qortal.data.arbitrary.ArbitraryResourceNameInfo; +import org.qortal.data.network.ArbitraryPeerData; import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.TransactionData; -import org.qortal.data.transaction.ArbitraryTransactionData.DataType; import org.qortal.repository.ArbitraryRepository; import org.qortal.repository.DataException; -import org.qortal.settings.Settings; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.transaction.Transaction.ApprovalStatus; import org.qortal.utils.Base58; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + public class HSQLDBArbitraryRepository implements ArbitraryRepository { + private static final Logger LOGGER = LogManager.getLogger(HSQLDBArbitraryRepository.class); + private static final int MAX_RAW_DATA_SIZE = 255; // size of VARBINARY protected HSQLDBRepository repository; - + public HSQLDBArbitraryRepository(HSQLDBRepository repository) { this.repository = repository; } - /** - * Returns pathname for saving arbitrary transaction data payloads. - *

- * Format: arbitrary//.raw - * - * @param arbitraryTransactionData - * @return - */ - public static String buildPathname(ArbitraryTransactionData arbitraryTransactionData) { - String senderAddress = Crypto.toAddress(arbitraryTransactionData.getSenderPublicKey()); - - StringBuilder stringBuilder = new StringBuilder(1024); - - stringBuilder.append(Settings.getInstance().getUserPath()); - stringBuilder.append("arbitrary"); - stringBuilder.append(File.separator); - stringBuilder.append(senderAddress); - stringBuilder.append(File.separator); - stringBuilder.append(arbitraryTransactionData.getService()); - stringBuilder.append(File.separator); - stringBuilder.append(Base58.encode(arbitraryTransactionData.getSignature())); - stringBuilder.append(".raw"); - - return stringBuilder.toString(); - } - private ArbitraryTransactionData getTransactionData(byte[] signature) throws DataException { TransactionData transactionData = this.repository.getTransactionRepository().fromSignature(signature); if (transactionData == null) @@ -64,99 +46,602 @@ private ArbitraryTransactionData getTransactionData(byte[] signature) throws Dat @Override public boolean isDataLocal(byte[] signature) throws DataException { ArbitraryTransactionData transactionData = getTransactionData(signature); - if (transactionData == null) + if (transactionData == null) { return false; + } // Raw data is always available - if (transactionData.getDataType() == DataType.RAW_DATA) + if (transactionData.getDataType() == DataType.RAW_DATA) { return true; + } - String dataPathname = buildPathname(transactionData); - - Path dataPath = Paths.get(dataPathname); - return Files.exists(dataPath); - } + // Load hashes + byte[] hash = transactionData.getData(); + byte[] metadataHash = transactionData.getMetadataHash(); - @Override - public byte[] fetchData(byte[] signature) throws DataException { - ArbitraryTransactionData transactionData = getTransactionData(signature); - if (transactionData == null) - return null; + // Load data file(s) + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); + arbitraryDataFile.setMetadataHash(metadataHash); - // Raw data is always available - if (transactionData.getDataType() == DataType.RAW_DATA) - return transactionData.getData(); + // Check if we already have the complete data file or all chunks + if (arbitraryDataFile.allFilesExist()) { + return true; + } - String dataPathname = buildPathname(transactionData); + return false; + } - Path dataPath = Paths.get(dataPathname); + @Override + public byte[] fetchData(byte[] signature) { try { - return Files.readAllBytes(dataPath); - } catch (IOException e) { + ArbitraryTransactionData transactionData = getTransactionData(signature); + if (transactionData == null) { + return null; + } + + // Raw data is always available + if (transactionData.getDataType() == DataType.RAW_DATA) { + return transactionData.getData(); + } + + // Load hashes + byte[] digest = transactionData.getData(); + byte[] metadataHash = transactionData.getMetadataHash(); + + // Load data file(s) + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + + // If we have the complete data file, return it + if (arbitraryDataFile.exists()) { + // Ensure the file's size matches the size reported by the transaction (throws a DataException if not) + arbitraryDataFile.validateFileSize(transactionData.getSize()); + + return arbitraryDataFile.getBytes(); + } + + // Alternatively, if we have all the chunks, combine them into a single file + if (arbitraryDataFile.allChunksExist()) { + arbitraryDataFile.join(); + + // Verify that the combined hash matches the expected hash + if (!digest.equals(arbitraryDataFile.digest())) { + LOGGER.info(String.format("Hash mismatch for transaction: %s", Base58.encode(signature))); + return null; + } + + // Ensure the file's size matches the size reported by the transaction + arbitraryDataFile.validateFileSize(transactionData.getSize()); + + return arbitraryDataFile.getBytes(); + } + + } catch (DataException e) { + LOGGER.info("Unable to fetch data for transaction {}: {}", Base58.encode(signature), e.getMessage()); return null; } + + return null; } @Override public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException { // Already hashed? Nothing to do - if (arbitraryTransactionData.getDataType() == DataType.DATA_HASH) + if (arbitraryTransactionData.getDataType() == DataType.DATA_HASH) { return; + } // Trivial-sized payloads can remain in raw form - if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA && arbitraryTransactionData.getData().length <= MAX_RAW_DATA_SIZE) + if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA && arbitraryTransactionData.getData().length <= MAX_RAW_DATA_SIZE) { return; + } - // Store non-trivial payloads in filesystem and convert transaction's data to hash form - byte[] rawData = arbitraryTransactionData.getData(); + throw new IllegalStateException(String.format("Supplied data is larger than maximum size (%d bytes). Please use ArbitraryDataWriter.", MAX_RAW_DATA_SIZE)); + } - // Calculate hash of data and update our transaction to use that - byte[] dataHash = Crypto.digest(rawData); - arbitraryTransactionData.setData(dataHash); - arbitraryTransactionData.setDataType(DataType.DATA_HASH); + @Override + public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException { + // No need to do anything if we still only have raw data, and hence nothing saved in filesystem + if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA) { + return; + } - String dataPathname = buildPathname(arbitraryTransactionData); + // Load hashes + byte[] hash = arbitraryTransactionData.getData(); + byte[] metadataHash = arbitraryTransactionData.getMetadataHash(); - Path dataPath = Paths.get(dataPathname); + // Load data file(s) + byte[] signature = arbitraryTransactionData.getSignature(); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); + arbitraryDataFile.setMetadataHash(metadataHash); - // Make sure directory structure exists - try { - Files.createDirectories(dataPath.getParent()); - } catch (IOException e) { - throw new DataException("Unable to create arbitrary transaction directory", e); + // Delete file and chunks + arbitraryDataFile.deleteAll(); + } + + @Override + public List getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException { + String sql = "SELECT type, reference, signature, creator, created_when, fee, " + + "tx_group_id, block_height, approval_status, approval_height, " + + "version, nonce, service, size, is_data_raw, data, metadata_hash, " + + "name, identifier, update_method, secret, compression FROM ArbitraryTransactions " + + "JOIN Transactions USING (signature) " + + "WHERE lower(name) = ? AND service = ?" + + "AND (identifier = ? OR (identifier IS NULL AND ? IS NULL))" + + "AND created_when >= ? ORDER BY created_when ASC"; + List arbitraryTransactionData = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql, name.toLowerCase(), service.value, identifier, identifier, since)) { + if (resultSet == null) + return null; + + do { + //TransactionType type = TransactionType.valueOf(resultSet.getInt(1)); + + byte[] reference = resultSet.getBytes(2); + byte[] signature = resultSet.getBytes(3); + byte[] creatorPublicKey = resultSet.getBytes(4); + long timestamp = resultSet.getLong(5); + + Long fee = resultSet.getLong(6); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(7); + + Integer blockHeight = resultSet.getInt(8); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9)); + Integer approvalHeight = resultSet.getInt(10); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + int version = resultSet.getInt(11); + int nonce = resultSet.getInt(12); + Service serviceResult = Service.valueOf(resultSet.getInt(13)); + int size = resultSet.getInt(14); + boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false + DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; + byte[] data = resultSet.getBytes(16); + byte[] metadataHash = resultSet.getBytes(17); + String nameResult = resultSet.getString(18); + String identifierResult = resultSet.getString(19); + Method method = Method.valueOf(resultSet.getInt(20)); + byte[] secret = resultSet.getBytes(21); + Compression compression = Compression.valueOf(resultSet.getInt(22)); + // FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls. + + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + version, serviceResult, nonce, size, nameResult, identifierResult, method, secret, + compression, data, dataType, metadataHash, null); + + arbitraryTransactionData.add(transactionData); + } while (resultSet.next()); + + return arbitraryTransactionData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); } + } - // Output actual transaction data - try (OutputStream dataOut = Files.newOutputStream(dataPath)) { - dataOut.write(rawData); - } catch (IOException e) { - throw new DataException("Unable to store arbitrary transaction data", e); + @Override + public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException { + StringBuilder sql = new StringBuilder(1024); + + sql.append("SELECT type, reference, signature, creator, created_when, fee, " + + "tx_group_id, block_height, approval_status, approval_height, " + + "version, nonce, service, size, is_data_raw, data, metadata_hash, " + + "name, identifier, update_method, secret, compression FROM ArbitraryTransactions " + + "JOIN Transactions USING (signature) " + + "WHERE lower(name) = ? AND service = ? " + + "AND (identifier = ? OR (identifier IS NULL AND ? IS NULL))"); + + if (method != null) { + sql.append(" AND update_method = "); + sql.append(method.value); + } + + sql.append("ORDER BY created_when DESC LIMIT 1"); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value, identifier, identifier)) { + if (resultSet == null) + return null; + + //TransactionType type = TransactionType.valueOf(resultSet.getInt(1)); + + byte[] reference = resultSet.getBytes(2); + byte[] signature = resultSet.getBytes(3); + byte[] creatorPublicKey = resultSet.getBytes(4); + long timestamp = resultSet.getLong(5); + + Long fee = resultSet.getLong(6); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(7); + + Integer blockHeight = resultSet.getInt(8); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9)); + Integer approvalHeight = resultSet.getInt(10); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + int version = resultSet.getInt(11); + int nonce = resultSet.getInt(12); + Service serviceResult = Service.valueOf(resultSet.getInt(13)); + int size = resultSet.getInt(14); + boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false + DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; + byte[] data = resultSet.getBytes(16); + byte[] metadataHash = resultSet.getBytes(17); + String nameResult = resultSet.getString(18); + String identifierResult = resultSet.getString(19); + Method methodResult = Method.valueOf(resultSet.getInt(20)); + byte[] secret = resultSet.getBytes(21); + Compression compression = Compression.valueOf(resultSet.getInt(22)); + // FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls. + + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + version, serviceResult, nonce, size, nameResult, identifierResult, methodResult, secret, + compression, data, dataType, metadataHash, null); + + return transactionData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); } } @Override - public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException { - // No need to do anything if we still only have raw data, and hence nothing saved in filesystem - if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA) - return; + public List getArbitraryResources(Service service, String identifier, String name, + boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException { + StringBuilder sql = new StringBuilder(512); + List bindParams = new ArrayList<>(); + + sql.append("SELECT name, service, identifier, MAX(size) AS max_size FROM ArbitraryTransactions WHERE 1=1"); + + if (service != null) { + sql.append(" AND service = "); + sql.append(service.value); + } + + if (defaultResource) { + // Default resource requested - use NULL identifier + sql.append(" AND identifier IS NULL"); + } + else { + // Non-default resource requested + // Use an exact match identifier, or list all if supplied identifier is null + sql.append(" AND (identifier = ? OR (? IS NULL))"); + bindParams.add(identifier); + bindParams.add(identifier); + } + + if (name != null) { + sql.append(" AND name = ?"); + bindParams.add(name); + } + + sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); + + if (reverse != null && reverse) { + sql.append(" DESC"); + } + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List arbitraryResources = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return null; + + do { + String nameResult = resultSet.getString(1); + Service serviceResult = Service.valueOf(resultSet.getInt(2)); + String identifierResult = resultSet.getString(3); + Integer sizeResult = resultSet.getInt(4); + + // We should filter out resources without names + if (nameResult == null) { + continue; + } + + ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); + arbitraryResourceInfo.name = nameResult; + arbitraryResourceInfo.service = serviceResult; + arbitraryResourceInfo.identifier = identifierResult; + arbitraryResourceInfo.size = Longs.valueOf(sizeResult); + + arbitraryResources.add(arbitraryResourceInfo); + } while (resultSet.next()); + + return arbitraryResources; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); + } + } + + @Override + public List searchArbitraryResources(Service service, String query, + boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException { + StringBuilder sql = new StringBuilder(512); + List bindParams = new ArrayList<>(); + + // For now we are searching anywhere in the fields + // Note that this will bypass any indexes so may not scale well + // Longer term we probably want to copy resources to their own table anyway + String queryWildcard = String.format("%%%s%%", query.toLowerCase()); + + sql.append("SELECT name, service, identifier, MAX(size) AS max_size FROM ArbitraryTransactions WHERE 1=1"); + + if (service != null) { + sql.append(" AND service = "); + sql.append(service.value); + } + + if (defaultResource) { + // Default resource requested - use NULL identifier and search name only + sql.append(" AND LCASE(name) LIKE ? AND identifier IS NULL"); + bindParams.add(queryWildcard); + } + else { + // Non-default resource requested + // In this case we search the identifier as well as the name + sql.append(" AND (LCASE(name) LIKE ? OR LCASE(identifier) LIKE ?)"); + bindParams.add(queryWildcard); + bindParams.add(queryWildcard); + } + + sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); + + if (reverse != null && reverse) { + sql.append(" DESC"); + } + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List arbitraryResources = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return null; + + do { + String nameResult = resultSet.getString(1); + Service serviceResult = Service.valueOf(resultSet.getInt(2)); + String identifierResult = resultSet.getString(3); + Integer sizeResult = resultSet.getInt(4); + + // We should filter out resources without names + if (nameResult == null) { + continue; + } + + ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); + arbitraryResourceInfo.name = nameResult; + arbitraryResourceInfo.service = serviceResult; + arbitraryResourceInfo.identifier = identifierResult; + arbitraryResourceInfo.size = Longs.valueOf(sizeResult); + + arbitraryResources.add(arbitraryResourceInfo); + } while (resultSet.next()); + + return arbitraryResources; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); + } + } + + @Override + public List getArbitraryResourceCreatorNames(Service service, String identifier, + boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException { + StringBuilder sql = new StringBuilder(512); + + sql.append("SELECT name FROM ArbitraryTransactions WHERE 1=1"); + + if (service != null) { + sql.append(" AND service = "); + sql.append(service.value); + } + + if (defaultResource) { + // Default resource requested - use NULL identifier + // The AND ? IS NULL AND ? IS NULL is a hack to make use of the identifier params in checkedExecute() + identifier = null; + sql.append(" AND (identifier IS NULL AND ? IS NULL AND ? IS NULL)"); + } + else { + // Non-default resource requested + // Use an exact match identifier, or list all if supplied identifier is null + sql.append(" AND (identifier = ? OR (? IS NULL))"); + } + + sql.append(" GROUP BY name ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); + + if (reverse != null && reverse) { + sql.append(" DESC"); + } + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List arbitraryResources = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), identifier, identifier)) { + if (resultSet == null) + return null; + + do { + String name = resultSet.getString(1); + + // We should filter out resources without names + if (name == null) { + continue; + } + + ArbitraryResourceNameInfo arbitraryResourceNameInfo = new ArbitraryResourceNameInfo(); + arbitraryResourceNameInfo.name = name; + + arbitraryResources.add(arbitraryResourceNameInfo); + } while (resultSet.next()); + + return arbitraryResources; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); + } + } + + + // Peer file tracking + + /** + * Fetch a list of peers that have reported to be holding chunks related to + * supplied transaction signature. + * @param signature + * @return a list of ArbitraryPeerData objects, or null if none found + * @throws DataException + */ + @Override + public List getArbitraryPeerDataForSignature(byte[] signature) throws DataException { + // Hash the signature so it fits within 32 bytes + byte[] hashedSignature = Crypto.digest(signature); + + String sql = "SELECT hash, peer_address, successes, failures, last_attempted, last_retrieved " + + "FROM ArbitraryPeers " + + "WHERE hash = ?"; + + List arbitraryPeerData = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql, hashedSignature)) { + if (resultSet == null) + return null; + + do { + byte[] hash = resultSet.getBytes(1); + String peerAddr = resultSet.getString(2); + Integer successes = resultSet.getInt(3); + Integer failures = resultSet.getInt(4); + Long lastAttempted = resultSet.getLong(5); + Long lastRetrieved = resultSet.getLong(6); + + ArbitraryPeerData peerData = new ArbitraryPeerData(hash, peerAddr, successes, failures, + lastAttempted, lastRetrieved); + + arbitraryPeerData.add(peerData); + } while (resultSet.next()); + + return arbitraryPeerData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary peer data from repository", e); + } + } + + public ArbitraryPeerData getArbitraryPeerDataForSignatureAndPeer(byte[] signature, String peerAddress) throws DataException { + // Hash the signature so it fits within 32 bytes + byte[] hashedSignature = Crypto.digest(signature); + + String sql = "SELECT hash, peer_address, successes, failures, last_attempted, last_retrieved " + + "FROM ArbitraryPeers " + + "WHERE hash = ? AND peer_address = ?"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, hashedSignature, peerAddress)) { + if (resultSet == null) + return null; + + byte[] hash = resultSet.getBytes(1); + String peerAddr = resultSet.getString(2); + Integer successes = resultSet.getInt(3); + Integer failures = resultSet.getInt(4); + Long lastAttempted = resultSet.getLong(5); + Long lastRetrieved = resultSet.getLong(6); + + ArbitraryPeerData arbitraryPeerData = new ArbitraryPeerData(hash, peerAddr, successes, failures, + lastAttempted, lastRetrieved); + + return arbitraryPeerData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary peer data from repository", e); + } + } + + public ArbitraryPeerData getArbitraryPeerDataForSignatureAndHost(byte[] signature, String host) throws DataException { + // Hash the signature so it fits within 32 bytes + byte[] hashedSignature = Crypto.digest(signature); + + // Create a host wildcard string which allows any port + String hostWildcard = String.format("%s:%%", host); + + String sql = "SELECT hash, peer_address, successes, failures, last_attempted, last_retrieved " + + "FROM ArbitraryPeers " + + "WHERE hash = ? AND peer_address LIKE ?"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, hashedSignature, hostWildcard)) { + if (resultSet == null) + return null; + + byte[] hash = resultSet.getBytes(1); + String peerAddr = resultSet.getString(2); + Integer successes = resultSet.getInt(3); + Integer failures = resultSet.getInt(4); + Long lastAttempted = resultSet.getLong(5); + Long lastRetrieved = resultSet.getLong(6); + + ArbitraryPeerData arbitraryPeerData = new ArbitraryPeerData(hash, peerAddr, successes, failures, + lastAttempted, lastRetrieved); + + return arbitraryPeerData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary peer data from repository", e); + } + } + + @Override + public void save(ArbitraryPeerData arbitraryPeerData) throws DataException { + HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryPeers"); + + saveHelper.bind("hash", arbitraryPeerData.getHash()) + .bind("peer_address", arbitraryPeerData.getPeerAddress()) + .bind("successes", arbitraryPeerData.getSuccesses()) + .bind("failures", arbitraryPeerData.getFailures()) + .bind("last_attempted", arbitraryPeerData.getLastAttempted()) + .bind("last_retrieved", arbitraryPeerData.getLastRetrieved()); - String dataPathname = buildPathname(arbitraryTransactionData); - Path dataPath = Paths.get(dataPathname); try { - Files.deleteIfExists(dataPath); + saveHelper.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save ArbitraryPeerData into repository", e); + } + } - // Also attempt to delete parent directory if empty - Path servicePath = dataPath.getParent(); - Files.deleteIfExists(servicePath); + @Override + public void delete(ArbitraryPeerData arbitraryPeerData) throws DataException { + try { + // Remove peer/hash combination + this.repository.delete("ArbitraryPeers", "hash = ? AND peer_address = ?", + arbitraryPeerData.getHash(), arbitraryPeerData.getPeerAddress()); - // Also attempt to delete parent directory if empty - Path senderpath = servicePath.getParent(); - Files.deleteIfExists(senderpath); - } catch (DirectoryNotEmptyException e) { - // One of the parent service/sender directories still has data from other transactions - this is OK - } catch (IOException e) { - throw new DataException("Unable to delete arbitrary transaction data", e); + } catch (SQLException e) { + throw new DataException("Unable to delete arbitrary peer data from repository", e); } } + @Override + public void deleteArbitraryPeersWithSignature(byte[] signature) throws DataException { + byte[] hash = Crypto.digest(signature); + try { + // Remove all records of peers hosting supplied signature + this.repository.delete("ArbitraryPeers", "hash = ?", hash); + + } catch (SQLException e) { + throw new DataException("Unable to delete arbitrary peer data from repository", e); + } + } } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java new file mode 100644 index 000000000..d8738f0d3 --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java @@ -0,0 +1,310 @@ +package org.qortal.repository.hsqldb; + +import org.qortal.api.ApiError; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.model.BlockSignerSummary; +import org.qortal.block.Block; +import org.qortal.data.block.BlockArchiveData; +import org.qortal.data.block.BlockData; +import org.qortal.data.block.BlockSummaryData; +import org.qortal.repository.BlockArchiveReader; +import org.qortal.repository.BlockArchiveRepository; +import org.qortal.repository.DataException; +import org.qortal.utils.Triple; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository { + + protected HSQLDBRepository repository; + + public HSQLDBBlockArchiveRepository(HSQLDBRepository repository) { + this.repository = repository; + } + + + @Override + public BlockData fromSignature(byte[] signature) throws DataException { + Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockWithSignature(signature, this.repository); + if (blockInfo != null) { + return (BlockData) blockInfo.getA(); + } + return null; + } + + @Override + public int getHeightFromSignature(byte[] signature) throws DataException { + Integer height = BlockArchiveReader.getInstance().fetchHeightForSignature(signature, this.repository); + if (height == null || height == 0) { + return 0; + } + return height; + } + + @Override + public BlockData fromHeight(int height) throws DataException { + Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height); + if (blockInfo != null) { + return (BlockData) blockInfo.getA(); + } + return null; + } + + @Override + public List fromRange(int startHeight, int endHeight) throws DataException { + List blocks = new ArrayList<>(); + + for (int height = startHeight; height < endHeight; height++) { + BlockData blockData = this.fromHeight(height); + if (blockData == null) { + return blocks; + } + blocks.add(blockData); + } + return blocks; + } + + @Override + public BlockData fromReference(byte[] reference) throws DataException { + BlockData referenceBlock = this.repository.getBlockArchiveRepository().fromSignature(reference); + if (referenceBlock == null) { + // Try the main block repository. Needed for genesis block. + referenceBlock = this.repository.getBlockRepository().fromSignature(reference); + } + if (referenceBlock != null) { + int height = referenceBlock.getHeight(); + if (height > 0) { + // Request the block at height + 1 + Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height + 1); + if (blockInfo != null) { + return (BlockData) blockInfo.getA(); + } + } + } + return null; + } + + @Override + public int getHeightFromTimestamp(long timestamp) throws DataException { + String sql = "SELECT height FROM BlockArchive WHERE minted_when <= ? ORDER BY minted_when DESC, height DESC LIMIT 1"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, timestamp)) { + if (resultSet == null) { + return 0; + } + return resultSet.getInt(1); + + } catch (SQLException e) { + throw new DataException("Error fetching height from BlockArchive repository", e); + } + } + + @Override + public long getTimestampFromHeight(int height) throws DataException { + String sql = "SELECT minted_when FROM BlockArchive WHERE height = ?"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, height)) { + if (resultSet == null) + return 0; + + return resultSet.getLong(1); + } catch (SQLException e) { + throw new DataException("Error obtaining block timestamp by height from BlockArchive repository", e); + } + } + + @Override + public List getBlockSummariesBySigner(byte[] signerPublicKey, Integer limit, Integer offset, Boolean reverse) throws DataException { + StringBuilder sql = new StringBuilder(512); + sql.append("SELECT signature, height, BlockArchive.minter FROM "); + + // List of minter account's public key and reward-share public keys with minter's public key + sql.append("(SELECT * FROM (VALUES (CAST(? AS QortalPublicKey))) UNION (SELECT reward_share_public_key FROM RewardShares WHERE minter_public_key = ?)) AS PublicKeys (public_key) "); + + // Match BlockArchive blocks signed with public key from above list + sql.append("JOIN BlockArchive ON BlockArchive.minter = public_key "); + + sql.append("ORDER BY BlockArchive.height "); + if (reverse != null && reverse) + sql.append("DESC "); + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List blockSummaries = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), signerPublicKey, signerPublicKey)) { + if (resultSet == null) + return blockSummaries; + + do { + byte[] signature = resultSet.getBytes(1); + int height = resultSet.getInt(2); + byte[] blockMinterPublicKey = resultSet.getBytes(3); + + // Fetch additional info from the archive itself + int onlineAccountsCount = 0; + BlockData blockData = this.fromSignature(signature); + if (blockData != null) { + onlineAccountsCount = blockData.getOnlineAccountsCount(); + } + + BlockSummaryData blockSummary = new BlockSummaryData(height, signature, blockMinterPublicKey, onlineAccountsCount); + blockSummaries.add(blockSummary); + } while (resultSet.next()); + + return blockSummaries; + } catch (SQLException e) { + throw new DataException("Unable to fetch minter's block summaries from repository", e); + } + } + + @Override + public List getBlockSigners(List addresses, Integer limit, Integer offset, Boolean reverse) throws DataException { + String subquerySql = "SELECT minter, COUNT(signature) FROM (" + + "(SELECT minter, signature FROM Blocks) UNION ALL (SELECT minter, signature FROM BlockArchive)" + + ") GROUP BY minter"; + + StringBuilder sql = new StringBuilder(1024); + sql.append("SELECT DISTINCT block_minter, n_blocks, minter_public_key, minter, recipient FROM ("); + sql.append(subquerySql); + sql.append(") AS Minters (block_minter, n_blocks) LEFT OUTER JOIN RewardShares ON reward_share_public_key = block_minter "); + + if (addresses != null && !addresses.isEmpty()) { + sql.append(" LEFT OUTER JOIN Accounts AS BlockMinterAccounts ON BlockMinterAccounts.public_key = block_minter "); + sql.append(" LEFT OUTER JOIN Accounts AS RewardShareMinterAccounts ON RewardShareMinterAccounts.public_key = minter_public_key "); + sql.append(" JOIN (VALUES "); + + final int addressesSize = addresses.size(); + for (int ai = 0; ai < addressesSize; ++ai) { + if (ai != 0) + sql.append(", "); + + sql.append("(?)"); + } + + sql.append(") AS FilterAccounts (account) "); + sql.append(" ON FilterAccounts.account IN (recipient, BlockMinterAccounts.account, RewardShareMinterAccounts.account) "); + } else { + addresses = Collections.emptyList(); + } + + sql.append("ORDER BY n_blocks "); + if (reverse != null && reverse) + sql.append("DESC "); + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List summaries = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), addresses.toArray())) { + if (resultSet == null) + return summaries; + + do { + byte[] blockMinterPublicKey = resultSet.getBytes(1); + int nBlocks = resultSet.getInt(2); + + // May not be present if no reward-share: + byte[] mintingAccountPublicKey = resultSet.getBytes(3); + String minterAccount = resultSet.getString(4); + String recipientAccount = resultSet.getString(5); + + BlockSignerSummary blockSignerSummary; + if (recipientAccount == null) + blockSignerSummary = new BlockSignerSummary(blockMinterPublicKey, nBlocks); + else + blockSignerSummary = new BlockSignerSummary(blockMinterPublicKey, nBlocks, mintingAccountPublicKey, minterAccount, recipientAccount); + + summaries.add(blockSignerSummary); + } while (resultSet.next()); + + return summaries; + } catch (SQLException e) { + throw new DataException("Unable to fetch block minters from repository", e); + } + } + + + @Override + public int getBlockArchiveHeight() throws DataException { + String sql = "SELECT block_archive_height FROM DatabaseInfo"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return 0; + + return resultSet.getInt(1); + } catch (SQLException e) { + throw new DataException("Unable to fetch block archive height from repository", e); + } + } + + @Override + public void setBlockArchiveHeight(int archiveHeight) throws DataException { + // trimHeightsLock is to prevent concurrent update on DatabaseInfo + // that could result in "transaction rollback: serialization failure" + synchronized (this.repository.trimHeightsLock) { + String updateSql = "UPDATE DatabaseInfo SET block_archive_height = ?"; + + try { + this.repository.executeCheckedUpdate(updateSql, archiveHeight); + this.repository.saveChanges(); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to set block archive height in repository", e); + } + } + } + + + @Override + public BlockArchiveData getBlockArchiveDataForSignature(byte[] signature) throws DataException { + String sql = "SELECT height, signature, minted_when, minter FROM BlockArchive WHERE signature = ? LIMIT 1"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, signature)) { + if (resultSet == null) { + return null; + } + int height = resultSet.getInt(1); + byte[] sig = resultSet.getBytes(2); + long timestamp = resultSet.getLong(3); + byte[] minterPublicKey = resultSet.getBytes(4); + return new BlockArchiveData(sig, height, timestamp, minterPublicKey); + + } catch (SQLException e) { + throw new DataException("Error fetching height from BlockArchive repository", e); + } + } + + + @Override + public void save(BlockArchiveData blockArchiveData) throws DataException { + HSQLDBSaver saveHelper = new HSQLDBSaver("BlockArchive"); + + saveHelper.bind("signature", blockArchiveData.getSignature()) + .bind("height", blockArchiveData.getHeight()) + .bind("minted_when", blockArchiveData.getTimestamp()) + .bind("minter", blockArchiveData.getMinterPublicKey()); + + try { + saveHelper.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save SimpleBlockData into BlockArchive repository", e); + } + } + + @Override + public void delete(BlockArchiveData blockArchiveData) throws DataException { + try { + this.repository.delete("BlockArchive", + "block_signature = ?", blockArchiveData.getSignature()); + } catch (SQLException e) { + throw new DataException("Unable to delete SimpleBlockData from BlockArchive repository", e); + } + } + +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java index 0860e1b17..b82380855 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java @@ -10,6 +10,7 @@ import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.block.BlockTransactionData; +import org.qortal.data.block.BlockArchiveData; import org.qortal.data.transaction.TransactionData; import org.qortal.repository.BlockRepository; import org.qortal.repository.DataException; @@ -119,7 +120,7 @@ public int getHeightFromSignature(byte[] signature) throws DataException { @Override public int getHeightFromTimestamp(long timestamp) throws DataException { // Uses (minted_when, height) index - String sql = "SELECT height FROM Blocks WHERE minted_when <= ? ORDER BY minted_when DESC LIMIT 1"; + String sql = "SELECT height FROM Blocks WHERE minted_when <= ? ORDER BY minted_when DESC, height DESC LIMIT 1"; try (ResultSet resultSet = this.repository.checkedExecute(sql, timestamp)) { if (resultSet == null) @@ -131,6 +132,20 @@ public int getHeightFromTimestamp(long timestamp) throws DataException { } } + @Override + public long getTimestampFromHeight(int height) throws DataException { + String sql = "SELECT minted_when FROM Blocks WHERE height = ?"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, height)) { + if (resultSet == null) + return 0; + + return resultSet.getLong(1); + } catch (SQLException e) { + throw new DataException("Error obtaining block timestamp by height from repository", e); + } + } + @Override public int getBlockchainHeight() throws DataException { String sql = "SELECT height FROM Blocks ORDER BY height DESC LIMIT 1"; @@ -160,7 +175,11 @@ public BlockData getLastBlock() throws DataException { public List getTransactionsFromSignature(byte[] signature, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(256); - sql.append("SELECT transaction_signature FROM BlockTransactions WHERE block_signature = ? ORDER BY sequence"); + sql.append("SELECT transaction_signature FROM BlockTransactions WHERE block_signature = ? ORDER BY block_signature"); + if (reverse != null && reverse) + sql.append(" DESC"); + + sql.append(", sequence"); if (reverse != null && reverse) sql.append(" DESC"); @@ -336,7 +355,8 @@ public List getBlocks(int firstBlockHeight, int lastBlockHeight) thro @Override public List getBlockSummaries(int firstBlockHeight, int lastBlockHeight) throws DataException { - String sql = "SELECT signature, height, minter, online_accounts_count FROM Blocks WHERE height BETWEEN ? AND ?"; + String sql = "SELECT signature, height, minter, online_accounts_count, minted_when, transaction_count " + + "FROM Blocks WHERE height BETWEEN ? AND ?"; List blockSummaries = new ArrayList<>(); @@ -349,8 +369,11 @@ public List getBlockSummaries(int firstBlockHeight, int lastBl int height = resultSet.getInt(2); byte[] minterPublicKey = resultSet.getBytes(3); int onlineAccountsCount = resultSet.getInt(4); + long timestamp = resultSet.getLong(5); + int transactionCount = resultSet.getInt(6); - BlockSummaryData blockSummary = new BlockSummaryData(height, signature, minterPublicKey, onlineAccountsCount); + BlockSummaryData blockSummary = new BlockSummaryData(height, signature, minterPublicKey, onlineAccountsCount, + timestamp, transactionCount); blockSummaries.add(blockSummary); } while (resultSet.next()); @@ -361,25 +384,108 @@ public List getBlockSummaries(int firstBlockHeight, int lastBl } @Override - public int trimOldOnlineAccountsSignatures(long timestamp) throws DataException { - String sql = "UPDATE Blocks set online_accounts_signatures = NULL WHERE minted_when < ? AND online_accounts_signatures IS NOT NULL"; + public int getOnlineAccountsSignaturesTrimHeight() throws DataException { + String sql = "SELECT online_signatures_trim_height FROM DatabaseInfo"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return 0; + + return resultSet.getInt(1); + } catch (SQLException e) { + throw new DataException("Unable to fetch online accounts signatures trim height from repository", e); + } + } + + @Override + public void setOnlineAccountsSignaturesTrimHeight(int trimHeight) throws DataException { + // trimHeightsLock is to prevent concurrent update on DatabaseInfo + // that could result in "transaction rollback: serialization failure" + synchronized (this.repository.trimHeightsLock) { + String updateSql = "UPDATE DatabaseInfo SET online_signatures_trim_height = ?"; + + try { + this.repository.executeCheckedUpdate(updateSql, trimHeight); + this.repository.saveChanges(); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to set online accounts signatures trim height in repository", e); + } + } + } + + @Override + public int trimOldOnlineAccountsSignatures(int minHeight, int maxHeight) throws DataException { + // We're often called so no need to trim all blocks in one go. + // Limit updates to reduce CPU and memory load. + String sql = "UPDATE Blocks SET online_accounts_signatures = NULL " + + "WHERE online_accounts_signatures IS NOT NULL " + + "AND height BETWEEN ? AND ?"; try { - return this.repository.checkedExecuteUpdateCount(sql, timestamp); + return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight); } catch (SQLException e) { + repository.examineException(e); throw new DataException("Unable to trim old online accounts signatures in repository", e); } } + + @Override + public int getBlockPruneHeight() throws DataException { + String sql = "SELECT block_prune_height FROM DatabaseInfo"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return 0; + + return resultSet.getInt(1); + } catch (SQLException e) { + throw new DataException("Unable to fetch block prune height from repository", e); + } + } + + @Override + public void setBlockPruneHeight(int pruneHeight) throws DataException { + // trimHeightsLock is to prevent concurrent update on DatabaseInfo + // that could result in "transaction rollback: serialization failure" + synchronized (this.repository.trimHeightsLock) { + String updateSql = "UPDATE DatabaseInfo SET block_prune_height = ?"; + + try { + this.repository.executeCheckedUpdate(updateSql, pruneHeight); + this.repository.saveChanges(); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to set block prune height in repository", e); + } + } + } + + @Override + public int pruneBlocks(int minHeight, int maxHeight) throws DataException { + // Don't prune the genesis block + if (minHeight <= 1) { + minHeight = 2; + } + + try { + return this.repository.delete("Blocks", "height BETWEEN ? AND ?", minHeight, maxHeight); + } catch (SQLException e) { + throw new DataException("Unable to prune blocks from repository", e); + } + } + + @Override - public BlockData getDetachedBlockSignature() throws DataException { + public BlockData getDetachedBlockSignature(int startHeight) throws DataException { String sql = "SELECT " + BLOCK_DB_COLUMNS + " FROM Blocks " + "LEFT OUTER JOIN Blocks AS ParentBlocks " + "ON ParentBlocks.signature = Blocks.reference " - + "WHERE ParentBlocks.signature IS NULL AND Blocks.height > 1 " + + "WHERE ParentBlocks.signature IS NULL AND Blocks.height > ? " + "ORDER BY Blocks.height ASC LIMIT 1"; - try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + try (ResultSet resultSet = this.repository.checkedExecute(sql, startHeight)) { return getBlockFromResultSet(resultSet); } catch (SQLException e) { throw new DataException("Error fetching block by signature from repository", e); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCrossChainRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCrossChainRepository.java new file mode 100644 index 000000000..29f2994ca --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCrossChainRepository.java @@ -0,0 +1,202 @@ +package org.qortal.repository.hsqldb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.repository.CrossChainRepository; +import org.qortal.repository.DataException; + +public class HSQLDBCrossChainRepository implements CrossChainRepository { + + protected HSQLDBRepository repository; + + public HSQLDBCrossChainRepository(HSQLDBRepository repository) { + this.repository = repository; + } + + @Override + public TradeBotData getTradeBotData(byte[] tradePrivateKey) throws DataException { + String sql = "SELECT acct_name, trade_state, trade_state_value, " + + "creator_address, at_address, " + + "updated_when, qort_amount, " + + "trade_native_public_key, trade_native_public_key_hash, " + + "trade_native_address, secret, hash_of_secret, " + + "foreign_blockchain, trade_foreign_public_key, trade_foreign_public_key_hash, " + + "foreign_amount, foreign_key, last_transaction_signature, locktime_a, receiving_account_info " + + "FROM TradeBotStates " + + "WHERE trade_private_key = ?"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, tradePrivateKey)) { + if (resultSet == null) + return null; + + String acctName = resultSet.getString(1); + String tradeState = resultSet.getString(2); + int tradeStateValue = resultSet.getInt(3); + String creatorAddress = resultSet.getString(4); + String atAddress = resultSet.getString(5); + long timestamp = resultSet.getLong(6); + long qortAmount = resultSet.getLong(7); + byte[] tradeNativePublicKey = resultSet.getBytes(8); + byte[] tradeNativePublicKeyHash = resultSet.getBytes(9); + String tradeNativeAddress = resultSet.getString(10); + byte[] secret = resultSet.getBytes(11); + byte[] hashOfSecret = resultSet.getBytes(12); + String foreignBlockchain = resultSet.getString(13); + byte[] tradeForeignPublicKey = resultSet.getBytes(14); + byte[] tradeForeignPublicKeyHash = resultSet.getBytes(15); + long foreignAmount = resultSet.getLong(16); + String foreignKey = resultSet.getString(17); + byte[] lastTransactionSignature = resultSet.getBytes(18); + Integer lockTimeA = resultSet.getInt(19); + if (lockTimeA == 0 && resultSet.wasNull()) + lockTimeA = null; + byte[] receivingAccountInfo = resultSet.getBytes(20); + + return new TradeBotData(tradePrivateKey, acctName, + tradeState, tradeStateValue, + creatorAddress, atAddress, timestamp, qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secret, hashOfSecret, + foreignBlockchain, tradeForeignPublicKey, tradeForeignPublicKeyHash, + foreignAmount, foreignKey, lastTransactionSignature, lockTimeA, receivingAccountInfo); + } catch (SQLException e) { + throw new DataException("Unable to fetch trade-bot trading state from repository", e); + } + } + + @Override + public boolean existsTradeWithAtExcludingStates(String atAddress, List excludeStates) throws DataException { + if (excludeStates == null) + excludeStates = Collections.emptyList(); + + StringBuilder whereClause = new StringBuilder(256); + whereClause.append("at_address = ?"); + + Object[] bindParams = new Object[1 + excludeStates.size()]; + bindParams[0] = atAddress; + + if (!excludeStates.isEmpty()) { + whereClause.append(" AND trade_state NOT IN (?"); + bindParams[1] = excludeStates.get(0); + + for (int i = 1; i < excludeStates.size(); ++i) { + whereClause.append(", ?"); + bindParams[1 + i] = excludeStates.get(i); + } + + whereClause.append(")"); + } + + try { + return this.repository.exists("TradeBotStates", whereClause.toString(), bindParams); + } catch (SQLException e) { + throw new DataException("Unable to check for trade-bot state in repository", e); + } + } + + @Override + public List getAllTradeBotData() throws DataException { + String sql = "SELECT trade_private_key, acct_name, trade_state, trade_state_value, " + + "creator_address, at_address, " + + "updated_when, qort_amount, " + + "trade_native_public_key, trade_native_public_key_hash, " + + "trade_native_address, secret, hash_of_secret, " + + "foreign_blockchain, trade_foreign_public_key, trade_foreign_public_key_hash, " + + "foreign_amount, foreign_key, last_transaction_signature, locktime_a, receiving_account_info " + + "FROM TradeBotStates"; + + List allTradeBotData = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return allTradeBotData; + + do { + byte[] tradePrivateKey = resultSet.getBytes(1); + String acctName = resultSet.getString(2); + String tradeState = resultSet.getString(3); + int tradeStateValue = resultSet.getInt(4); + String creatorAddress = resultSet.getString(5); + String atAddress = resultSet.getString(6); + long timestamp = resultSet.getLong(7); + long qortAmount = resultSet.getLong(8); + byte[] tradeNativePublicKey = resultSet.getBytes(9); + byte[] tradeNativePublicKeyHash = resultSet.getBytes(10); + String tradeNativeAddress = resultSet.getString(11); + byte[] secret = resultSet.getBytes(12); + byte[] hashOfSecret = resultSet.getBytes(13); + String foreignBlockchain = resultSet.getString(14); + byte[] tradeForeignPublicKey = resultSet.getBytes(15); + byte[] tradeForeignPublicKeyHash = resultSet.getBytes(16); + long foreignAmount = resultSet.getLong(17); + String foreignKey = resultSet.getString(18); + byte[] lastTransactionSignature = resultSet.getBytes(19); + Integer lockTimeA = resultSet.getInt(20); + if (lockTimeA == 0 && resultSet.wasNull()) + lockTimeA = null; + byte[] receivingAccountInfo = resultSet.getBytes(21); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, acctName, + tradeState, tradeStateValue, + creatorAddress, atAddress, timestamp, qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secret, hashOfSecret, + foreignBlockchain, tradeForeignPublicKey, tradeForeignPublicKeyHash, + foreignAmount, foreignKey, lastTransactionSignature, lockTimeA, receivingAccountInfo); + allTradeBotData.add(tradeBotData); + } while (resultSet.next()); + + return allTradeBotData; + } catch (SQLException e) { + throw new DataException("Unable to fetch trade-bot trading states from repository", e); + } + } + + @Override + public void save(TradeBotData tradeBotData) throws DataException { + HSQLDBSaver saveHelper = new HSQLDBSaver("TradeBotStates"); + + saveHelper.bind("trade_private_key", tradeBotData.getTradePrivateKey()) + .bind("acct_name", tradeBotData.getAcctName()) + .bind("trade_state", tradeBotData.getState()) + .bind("trade_state_value", tradeBotData.getStateValue()) + .bind("creator_address", tradeBotData.getCreatorAddress()) + .bind("at_address", tradeBotData.getAtAddress()) + .bind("updated_when", tradeBotData.getTimestamp()) + .bind("qort_amount", tradeBotData.getQortAmount()) + .bind("trade_native_public_key", tradeBotData.getTradeNativePublicKey()) + .bind("trade_native_public_key_hash", tradeBotData.getTradeNativePublicKeyHash()) + .bind("trade_native_address", tradeBotData.getTradeNativeAddress()) + .bind("secret", tradeBotData.getSecret()) + .bind("hash_of_secret", tradeBotData.getHashOfSecret()) + .bind("foreign_blockchain", tradeBotData.getForeignBlockchain()) + .bind("trade_foreign_public_key", tradeBotData.getTradeForeignPublicKey()) + .bind("trade_foreign_public_key_hash", tradeBotData.getTradeForeignPublicKeyHash()) + .bind("foreign_amount", tradeBotData.getForeignAmount()) + .bind("foreign_key", tradeBotData.getForeignKey()) + .bind("last_transaction_signature", tradeBotData.getLastTransactionSignature()) + .bind("locktime_a", tradeBotData.getLockTimeA()) + .bind("receiving_account_info", tradeBotData.getReceivingAccountInfo()); + + try { + saveHelper.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save trade bot data into repository", e); + } + } + + @Override + public int delete(byte[] tradePrivateKey) throws DataException { + try { + return this.repository.delete("TradeBotStates", "trade_private_key = ?", tradePrivateKey); + } catch (SQLException e) { + throw new DataException("Unable to delete trade-bot states from repository", e); + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java new file mode 100644 index 000000000..77136ab9c --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java @@ -0,0 +1,88 @@ +package org.qortal.repository.hsqldb; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.gui.SplashFrame; +import org.qortal.repository.BlockArchiveWriter; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.transform.TransformationException; + +import java.io.IOException; + +/** + * + * When switching to an archiving node, we need to archive most of the database contents. + * This involves copying its data into flat files. + * If we do this entirely as a background process, it is very slow and can interfere with syncing. + * However, if we take the approach of doing this in bulk, before starting up the rest of the + * processes, this makes it much faster and less invasive. + * + * From that point, the original background archiving process will run, but can be dialled right down + * so not to interfere with syncing. + * + */ + + +public class HSQLDBDatabaseArchiving { + + private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabaseArchiving.class); + + + public static boolean buildBlockArchive(Repository repository, long fileSizeTarget) throws DataException { + + // Only build the archive if we haven't already got one that is up to date + boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository); + if (upToDate) { + // Already archived + return false; + } + + LOGGER.info("Building block archive - this process could take a while... (approx. 15 mins on high spec)"); + SplashFrame.getInstance().updateStatus("Building block archive (takes 60+ mins)..."); + + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + int startHeight = 0; + + while (!Controller.isStopping()) { + try { + BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository); + writer.setFileSizeTarget(fileSizeTarget); + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + switch (result) { + case OK: + // Increment block archive height + startHeight = writer.getLastWrittenHeight() + 1; + repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight); + repository.saveChanges(); + break; + + case STOPPING: + return false; + + case NOT_ENOUGH_BLOCKS: + // We've reached the limit of the blocks we can archive + // Return from the whole method + return true; + + case BLOCK_NOT_FOUND: + // We tried to archive a block that didn't exist. This is a major failure and likely means + // that a bootstrap or re-sync is needed. Return rom the method + LOGGER.info("Error: block not found when building archive. If this error persists, " + + "a bootstrap or re-sync may be needed."); + return false; + } + + } catch (IOException | TransformationException | InterruptedException e) { + LOGGER.info("Caught exception when creating block cache", e); + return false; + } + } + + // If we got this far then something went wrong (most likely the app is stopping) + return false; + } + +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java new file mode 100644 index 000000000..978ba25eb --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java @@ -0,0 +1,332 @@ +package org.qortal.repository.hsqldb; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.data.block.BlockData; +import org.qortal.gui.SplashFrame; +import org.qortal.repository.BlockArchiveWriter; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.settings.Settings; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.concurrent.TimeoutException; + +/** + * + * When switching from a full node to a pruning node, we need to delete most of the database contents. + * If we do this entirely as a background process, it is very slow and can interfere with syncing. + * However, if we take the approach of transferring only the necessary rows to a new table and then + * deleting the original table, this makes the process much faster. It was taking several days to + * delete the AT states in the background, but only a couple of minutes to copy them to a new table. + * + * The trade off is that we have to go through a form of "reshape" when starting the app for the first + * time after enabling pruning mode. But given that this is an opt-in mode, I don't think it will be + * a problem. + * + * Once the pruning is complete, it automatically performs a CHECKPOINT DEFRAG in order to + * shrink the database file size down to a fraction of what it was before. + * + * From this point, the original background process will run, but can be dialled right down so not + * to interfere with syncing. + * + */ + + +public class HSQLDBDatabasePruning { + + private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabasePruning.class); + + + public static boolean pruneATStates(HSQLDBRepository repository) throws SQLException, DataException { + + // Only bulk prune AT states if we have never done so before + int pruneHeight = repository.getATRepository().getAtPruneHeight(); + if (pruneHeight > 0) { + // Already pruned AT states + return false; + } + + if (Settings.getInstance().isArchiveEnabled()) { + // Only proceed if we can see that the archiver has already finished + // This way, if the archiver failed for any reason, we can prune once it has had + // some opportunities to try again + boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository); + if (!upToDate) { + return false; + } + } + + LOGGER.info("Starting bulk prune of AT states - this process could take a while... " + + "(approx. 2 mins on high spec, or upwards of 30 mins in some cases)"); + SplashFrame.getInstance().updateStatus("Pruning database (takes up to 30 mins)..."); + + // Create new AT-states table to hold smaller dataset + repository.executeCheckedUpdate("DROP TABLE IF EXISTS ATStatesNew"); + repository.executeCheckedUpdate("CREATE TABLE ATStatesNew (" + + "AT_address QortalAddress, height INTEGER NOT NULL, state_hash ATStateHash NOT NULL, " + + "fees QortalAmount NOT NULL, is_initial BOOLEAN NOT NULL, sleep_until_message_timestamp BIGINT, " + + "PRIMARY KEY (AT_address, height), " + + "FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)"); + repository.executeCheckedUpdate("SET TABLE ATStatesNew NEW SPACE"); + repository.executeCheckedUpdate("CHECKPOINT"); + + // Add a height index + LOGGER.info("Adding index to AT states table..."); + repository.executeCheckedUpdate("CREATE INDEX IF NOT EXISTS ATStatesNewHeightIndex ON ATStatesNew (height)"); + repository.executeCheckedUpdate("CHECKPOINT"); + + + // Find our latest block + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + if (latestBlock == null) { + LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning"); + return false; + } + + // Calculate some constants for later use + final int blockchainHeight = latestBlock.getHeight(); + int maximumBlockToTrim = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + if (Settings.getInstance().isArchiveEnabled()) { + // Archive mode - don't prune anything that hasn't been archived yet + maximumBlockToTrim = Math.min(maximumBlockToTrim, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1); + } + final int endHeight = blockchainHeight; + final int blockStep = 10000; + + + // It's essential that we rebuild the latest AT states here, as we are using this data in the next query. + // Failing to do this will result in important AT states being deleted, rendering the database unusable. + repository.getATRepository().rebuildLatestAtStates(); + + + // Loop through all the LatestATStates and copy them to the new table + LOGGER.info("Copying AT states..."); + for (int height = 0; height < endHeight; height += blockStep) { + final int batchEndHeight = height + blockStep - 1; + //LOGGER.info(String.format("Copying AT states between %d and %d...", height, batchEndHeight)); + + String sql = "SELECT height, AT_address FROM LatestATStates WHERE height BETWEEN ? AND ?"; + try (ResultSet latestAtStatesResultSet = repository.checkedExecute(sql, height, batchEndHeight)) { + if (latestAtStatesResultSet != null) { + do { + int latestAtHeight = latestAtStatesResultSet.getInt(1); + String latestAtAddress = latestAtStatesResultSet.getString(2); + + // Copy this latest ATState to the new table + //LOGGER.info(String.format("Copying AT %s at height %d...", latestAtAddress, latestAtHeight)); + try { + String updateSql = "INSERT INTO ATStatesNew (" + + "SELECT AT_address, height, state_hash, fees, is_initial, sleep_until_message_timestamp " + + "FROM ATStates " + + "WHERE height = ? AND AT_address = ?)"; + repository.executeCheckedUpdate(updateSql, latestAtHeight, latestAtAddress); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to copy ATStates", e); + } + + // If this batch includes blocks after the maximum block to trim, we will need to copy + // each of its AT states above maximumBlockToTrim as they are considered "recent". We + // need to do this for _all_ AT states in these blocks, regardless of their latest state. + if (batchEndHeight >= maximumBlockToTrim) { + // Now copy this AT's states for each recent block they are present in + for (int i = maximumBlockToTrim; i < endHeight; i++) { + if (latestAtHeight < i) { + // This AT finished before this block so there is nothing to copy + continue; + } + + //LOGGER.info(String.format("Copying recent AT %s at height %d...", latestAtAddress, i)); + try { + // Copy each LatestATState to the new table + String updateSql = "INSERT IGNORE INTO ATStatesNew (" + + "SELECT AT_address, height, state_hash, fees, is_initial, sleep_until_message_timestamp " + + "FROM ATStates " + + "WHERE height = ? AND AT_address = ?)"; + repository.executeCheckedUpdate(updateSql, i, latestAtAddress); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to copy ATStates", e); + } + } + } + repository.saveChanges(); + + } while (latestAtStatesResultSet.next()); + } + } catch (SQLException e) { + throw new DataException("Unable to copy AT states", e); + } + } + + + // Finally, drop the original table and rename + LOGGER.info("Deleting old AT states..."); + repository.executeCheckedUpdate("DROP TABLE ATStates"); + repository.executeCheckedUpdate("ALTER TABLE ATStatesNew RENAME TO ATStates"); + repository.executeCheckedUpdate("ALTER INDEX ATStatesNewHeightIndex RENAME TO ATStatesHeightIndex"); + repository.executeCheckedUpdate("CHECKPOINT"); + + // Update the prune height + int nextPruneHeight = maximumBlockToTrim + 1; + repository.getATRepository().setAtPruneHeight(nextPruneHeight); + repository.saveChanges(); + + repository.executeCheckedUpdate("CHECKPOINT"); + + // Now prune/trim the ATStatesData, as this currently goes back over a month + return HSQLDBDatabasePruning.pruneATStateData(repository); + } + + /* + * Bulk prune ATStatesData to catch up with the now pruned ATStates table + * This uses the existing AT States trimming code but with a much higher end block + */ + private static boolean pruneATStateData(Repository repository) throws DataException { + + if (Settings.getInstance().isArchiveEnabled()) { + // Don't prune ATStatesData in archive mode + return true; + } + + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + if (latestBlock == null) { + LOGGER.info("Unable to determine blockchain height, necessary for bulk ATStatesData pruning"); + return false; + } + final int blockchainHeight = latestBlock.getHeight(); + int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + // ATStateData is already trimmed - so carry on from where we left off in the past + int pruneStartHeight = repository.getATRepository().getAtTrimHeight(); + + LOGGER.info("Starting bulk prune of AT states data - this process could take a while... (approx. 3 mins on high spec)"); + + while (pruneStartHeight < upperPrunableHeight) { + // Prune all AT state data up until our latest minus pruneBlockLimit (or our archive height) + + if (Controller.isStopping()) { + return false; + } + + // Override batch size in the settings because this is a one-off process + final int batchSize = 1000; + final int rowLimitPerBatch = 50000; + int upperBatchHeight = pruneStartHeight + batchSize; + int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); + + LOGGER.trace(String.format("Pruning AT states data between %d and %d...", pruneStartHeight, upperPruneHeight)); + + int numATStatesPruned = repository.getATRepository().trimAtStates(pruneStartHeight, upperPruneHeight, rowLimitPerBatch); + repository.saveChanges(); + + if (numATStatesPruned > 0) { + LOGGER.trace(String.format("Pruned %d AT states data rows between blocks %d and %d", + numATStatesPruned, pruneStartHeight, upperPruneHeight)); + } else { + repository.getATRepository().setAtTrimHeight(upperBatchHeight); + // No need to rebuild the latest AT states as we aren't currently synchronizing + repository.saveChanges(); + LOGGER.debug(String.format("Bumping AT states trim height to %d", upperBatchHeight)); + + // Can we move onto next batch? + if (upperPrunableHeight > upperBatchHeight) { + pruneStartHeight = upperBatchHeight; + } + else { + // We've finished pruning + break; + } + } + } + + return true; + } + + public static boolean pruneBlocks(Repository repository) throws SQLException, DataException { + + // Only bulk prune AT states if we have never done so before + int pruneHeight = repository.getBlockRepository().getBlockPruneHeight(); + if (pruneHeight > 0) { + // Already pruned blocks + return false; + } + + if (Settings.getInstance().isArchiveEnabled()) { + // Only proceed if we can see that the archiver has already finished + // This way, if the archiver failed for any reason, we can prune once it has had + // some opportunities to try again + boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository); + if (!upToDate) { + return false; + } + } + + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + if (latestBlock == null) { + LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning"); + return false; + } + final int blockchainHeight = latestBlock.getHeight(); + int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + int pruneStartHeight = 0; + + if (Settings.getInstance().isArchiveEnabled()) { + // Archive mode - don't prune anything that hasn't been archived yet + upperPrunableHeight = Math.min(upperPrunableHeight, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1); + } + + LOGGER.info("Starting bulk prune of blocks - this process could take a while... (approx. 5 mins on high spec)"); + + while (pruneStartHeight < upperPrunableHeight) { + // Prune all blocks up until our latest minus pruneBlockLimit + + int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize(); + int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); + + LOGGER.info(String.format("Pruning blocks between %d and %d...", pruneStartHeight, upperPruneHeight)); + + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(pruneStartHeight, upperPruneHeight); + repository.saveChanges(); + + if (numBlocksPruned > 0) { + LOGGER.info(String.format("Pruned %d block%s between %d and %d", + numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""), + pruneStartHeight, upperPruneHeight)); + } else { + final int nextPruneHeight = upperPruneHeight + 1; + repository.getBlockRepository().setBlockPruneHeight(nextPruneHeight); + repository.saveChanges(); + LOGGER.debug(String.format("Bumping block base prune height to %d", nextPruneHeight)); + + // Can we move onto next batch? + if (upperPrunableHeight > nextPruneHeight) { + pruneStartHeight = nextPruneHeight; + } + else { + // We've finished pruning + break; + } + } + } + + return true; + } + + public static void performMaintenance(Repository repository) throws SQLException, DataException { + try { + SplashFrame.getInstance().updateStatus("Performing maintenance..."); + + // Timeout if the database isn't ready for backing up after 5 minutes + // Nothing else should be using the db at this point, so a timeout shouldn't happen + long timeout = 5 * 60 * 1000L; + repository.performPeriodicMaintenance(timeout); + + } catch (TimeoutException e) { + LOGGER.info("Attempt to perform maintenance failed due to timeout: {}", e.getMessage()); + } + } + +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index c4751f448..9588aaa6e 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -4,9 +4,14 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.controller.tradebot.BitcoinACCTv1TradeBot; +import org.qortal.gui.SplashFrame; public class HSQLDBDatabaseUpdates { @@ -18,11 +23,21 @@ public class HSQLDBDatabaseUpdates { /** * Apply any incremental changes to database schema. * + * @return true if database was non-existent/empty, false otherwise * @throws SQLException */ - public static void updateDatabase(Connection connection) throws SQLException { - while (databaseUpdating(connection)) + public static boolean updateDatabase(Connection connection) throws SQLException { + final boolean wasPristine = fetchDatabaseVersion(connection) == 0; + + SplashFrame.getInstance().updateStatus("Upgrading database, please wait..."); + + while (databaseUpdating(connection, wasPristine)) incrementDatabaseVersion(connection); + + String text = String.format("Starting Qortal Core v%s...", Controller.getInstance().getVersionStringWithoutPrefix()); + SplashFrame.getInstance().updateStatus(text); + + return wasPristine; } /** @@ -40,23 +55,21 @@ private static void incrementDatabaseVersion(Connection connection) throws SQLEx /** * Fetch current version of database schema. * - * @return int, 0 if no schema yet + * @return database version, or 0 if no schema yet * @throws SQLException */ private static int fetchDatabaseVersion(Connection connection) throws SQLException { - int databaseVersion = 0; - try (Statement stmt = connection.createStatement()) { if (stmt.execute("SELECT version FROM DatabaseInfo")) try (ResultSet resultSet = stmt.getResultSet()) { if (resultSet.next()) - databaseVersion = resultSet.getInt(1); + return resultSet.getInt(1); } } catch (SQLException e) { // empty database } - return databaseVersion; + return 0; } /** @@ -65,7 +78,7 @@ private static int fetchDatabaseVersion(Connection connection) throws SQLExcepti * @return true - if a schema update happened, false otherwise * @throws SQLException */ - private static boolean databaseUpdating(Connection connection) throws SQLException { + private static boolean databaseUpdating(Connection connection, boolean wasPristine) throws SQLException { int databaseVersion = fetchDatabaseVersion(connection); try (Statement stmt = connection.createStatement()) { @@ -95,7 +108,6 @@ private static boolean databaseUpdating(Connection connection) throws SQLExcepti stmt.execute("CREATE COLLATION SQL_TEXT_NO_PAD FOR SQL_TEXT FROM SQL_TEXT NO PAD"); stmt.execute("SET FILES SPACE TRUE"); // Enable per-table block space within .data file, useful for CACHED table types - stmt.execute("SET FILES LOB SCALE 1"); // LOB granularity is 1KB // Slow down log fsync() calls from every 500ms to reduce I/O load stmt.execute("SET FILES WRITE DELAY 5"); // only fsync() every 5 seconds @@ -103,15 +115,15 @@ private static boolean databaseUpdating(Connection connection) throws SQLExcepti stmt.execute("INSERT INTO DatabaseInfo VALUES ( 0 )"); stmt.execute("CREATE TYPE ArbitraryData AS VARBINARY(256)"); - stmt.execute("CREATE TYPE AssetData AS CLOB(400K)"); + stmt.execute("CREATE TYPE AssetData AS VARCHAR(400K)"); stmt.execute("CREATE TYPE AssetID AS BIGINT"); stmt.execute("CREATE TYPE AssetName AS VARCHAR(34) COLLATE SQL_TEXT_NO_PAD"); stmt.execute("CREATE TYPE AssetOrderID AS VARBINARY(64)"); - stmt.execute("CREATE TYPE ATCode AS BLOB(64K)"); // 16bit * 1 - stmt.execute("CREATE TYPE ATCreationBytes AS BLOB(576K)"); // 16bit * 1 + 16bit * 8 + stmt.execute("CREATE TYPE ATCode AS VARBINARY(1024)"); // was: 16bit * 1 + stmt.execute("CREATE TYPE ATCreationBytes AS VARBINARY(4096)"); // was: 16bit * 1 + 16bit * 8 stmt.execute("CREATE TYPE ATMessage AS VARBINARY(32)"); stmt.execute("CREATE TYPE ATName AS VARCHAR(32) COLLATE SQL_TEXT_UCC_NO_PAD"); - stmt.execute("CREATE TYPE ATState AS BLOB(1M)"); // 16bit * 8 + 16bit * 4 + 16bit * 4 + stmt.execute("CREATE TYPE ATState AS VARBINARY(1024)"); // was: 16bit * 8 + 16bit * 4 + 16bit * 4 stmt.execute("CREATE TYPE ATTags AS VARCHAR(80) COLLATE SQL_TEXT_UCC_NO_PAD"); stmt.execute("CREATE TYPE ATType AS VARCHAR(32) COLLATE SQL_TEXT_UCC_NO_PAD"); stmt.execute("CREATE TYPE ATStateHash as VARBINARY(32)"); @@ -142,7 +154,7 @@ private static boolean databaseUpdating(Connection connection) throws SQLExcepti + "transaction_count INTEGER NOT NULL, total_fees QortalAmount NOT NULL, transactions_signature Signature NOT NULL, " + "height INTEGER NOT NULL, minted_when EpochMillis NOT NULL, " + "minter QortalPublicKey NOT NULL, minter_signature Signature NOT NULL, AT_count INTEGER NOT NULL, AT_fees QortalAmount NOT NULL, " - + "online_accounts BLOB(1M), online_accounts_count INTEGER NOT NULL, online_accounts_timestamp EpochMillis, online_accounts_signatures BLOB(1M), " + + "online_accounts VARBINARY(1024), online_accounts_count INTEGER NOT NULL, online_accounts_timestamp EpochMillis, online_accounts_signatures VARBINARY(1M), " + "PRIMARY KEY (signature))"); // For finding blocks by height. stmt.execute("CREATE INDEX BlockHeightIndex ON Blocks (height)"); @@ -154,16 +166,6 @@ private static boolean databaseUpdating(Connection connection) throws SQLExcepti stmt.execute("CREATE INDEX BlockTimestampHeightIndex ON Blocks (minted_when, height)"); // Use a separate table space as this table will be very large. stmt.execute("SET TABLE Blocks NEW SPACE"); - - // Table to hold next block height. - stmt.execute("CREATE TABLE NextBlockHeight (height INT NOT NULL)"); - // Initial value - should work for empty DB or populated DB. - stmt.execute("INSERT INTO NextBlockHeight VALUES (SELECT IFNULL(MAX(height), 0) + 1 FROM Blocks)"); - // We use triggers on Blocks to update a simple "next block height" table - String blockUpdateSql = "UPDATE NextBlockHeight SET height = (SELECT height + 1 FROM Blocks ORDER BY height DESC LIMIT 1)"; - stmt.execute("CREATE TRIGGER Next_block_height_insert_trigger AFTER INSERT ON Blocks " + blockUpdateSql); - stmt.execute("CREATE TRIGGER Next_block_height_update_trigger AFTER UPDATE ON Blocks " + blockUpdateSql); - stmt.execute("CREATE TRIGGER Next_block_height_delete_trigger AFTER DELETE ON Blocks " + blockUpdateSql); break; case 2: @@ -223,6 +225,8 @@ private static boolean databaseUpdating(Connection connection) throws SQLExcepti + "PRIMARY KEY (account))"); // For looking up an account by public key stmt.execute("CREATE INDEX AccountPublicKeyIndex on Accounts (public_key)"); + // Use a separate table space as this table will be very large. + stmt.execute("SET TABLE Accounts NEW SPACE"); // Account balances stmt.execute("CREATE TABLE AccountBalances (account QortalAddress, asset_id AssetID, balance QortalAmount NOT NULL, " @@ -231,6 +235,8 @@ private static boolean databaseUpdating(Connection connection) throws SQLExcepti stmt.execute("CREATE INDEX AccountBalancesAssetBalanceIndex ON AccountBalances (asset_id, balance)"); // Add CHECK constraint to account balances stmt.execute("ALTER TABLE AccountBalances ADD CONSTRAINT CheckBalanceNotNegative CHECK (balance >= 0)"); + // Use a separate table space as this table will be very large. + stmt.execute("SET TABLE AccountBalances NEW SPACE"); // Keeping track of QORT gained from holding legacy QORA stmt.execute("CREATE TABLE AccountQortFromQoraInfo (account QortalAddress, final_qort_from_qora QortalAmount, final_block_height INT, " @@ -280,7 +286,6 @@ private static boolean databaseUpdating(Connection connection) throws SQLExcepti + "service SMALLINT NOT NULL, is_data_raw BOOLEAN NOT NULL, data ArbitraryData NOT NULL, " + TRANSACTION_KEYS + ")"); // NB: Actual data payload stored elsewhere - // For the future: data payload should be encrypted, at the very least with transaction's reference as the seed for the encryption key break; case 8: @@ -428,6 +433,8 @@ private static boolean databaseUpdating(Connection connection) throws SQLExcepti + "PRIMARY KEY (AT_address, height), FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)"); // For finding per-block AT states, ordered by creation timestamp stmt.execute("CREATE INDEX BlockATStateIndex on ATStates (height, created_when)"); + // Use a separate table space as this table will be very large. + stmt.execute("SET TABLE ATStates NEW SPACE"); // Deploy CIYAM AT Transactions stmt.execute("CREATE TABLE DeployATTransactions (signature Signature, creator QortalPublicKey NOT NULL, AT_name ATName NOT NULL, " @@ -618,6 +625,340 @@ private static boolean databaseUpdating(Connection connection) throws SQLExcepti stmt.execute("CREATE TABLE PublicizeTransactions (signature Signature, nonce INT NOT NULL, " + TRANSACTION_KEYS + ")"); break; + case 20: + // Trade bot + // See case 25 below for changes + stmt.execute("CREATE TABLE TradeBotStates (trade_private_key QortalKeySeed NOT NULL, trade_state TINYINT NOT NULL, " + + "creator_address QortalAddress NOT NULL, at_address QortalAddress, updated_when BIGINT NOT NULL, qort_amount QortalAmount NOT NULL, " + + "trade_native_public_key QortalPublicKey NOT NULL, trade_native_public_key_hash VARBINARY(32) NOT NULL, " + + "trade_native_address QortalAddress NOT NULL, secret VARBINARY(32) NOT NULL, hash_of_secret VARBINARY(32) NOT NULL, " + + "trade_foreign_public_key VARBINARY(33) NOT NULL, trade_foreign_public_key_hash VARBINARY(32) NOT NULL, " + + "bitcoin_amount BIGINT NOT NULL, xprv58 VARCHAR(200), last_transaction_signature Signature, locktime_a BIGINT, " + + "receiving_account_info VARBINARY(32) NOT NULL, PRIMARY KEY (trade_private_key))"); + break; + + case 21: + // AT functionality index + stmt.execute("CREATE INDEX IF NOT EXISTS ATCodeHashIndex ON ATs (code_hash, is_finished)"); + break; + + case 22: + // LOB downsizing + stmt.execute("ALTER TABLE Blocks ALTER COLUMN online_accounts VARBINARY(1024)"); + stmt.execute("CHECKPOINT"); + stmt.execute("ALTER TABLE Blocks ALTER COLUMN online_accounts_signatures VARBINARY(1048576)"); + stmt.execute("CHECKPOINT"); + + stmt.execute("ALTER TABLE DeployATTransactions ALTER COLUMN creation_bytes VARBINARY(4096)"); + stmt.execute("CHECKPOINT"); + + stmt.execute("ALTER TABLE ATs ALTER COLUMN code_bytes VARBINARY(1024)"); + stmt.execute("CHECKPOINT"); + + stmt.execute("ALTER TABLE ATStates ALTER COLUMN state_data VARBINARY(1024)"); + stmt.execute("CHECKPOINT"); + break; + + case 23: + // MESSAGE transactions index + stmt.execute("CREATE INDEX IF NOT EXISTS MessageTransactionsRecipientIndex ON MessageTransactions (recipient, sender)"); + break; + + case 24: + // Remove unused NextBlockHeight table and corresponding triggers + stmt.execute("DROP TRIGGER IF EXISTS Next_block_height_insert_trigger"); + stmt.execute("DROP TRIGGER IF EXISTS Next_block_height_update_trigger"); + stmt.execute("DROP TRIGGER IF EXISTS Next_block_height_delete_trigger"); + stmt.execute("DROP TABLE IF EXISTS NextBlockHeight"); + break; + + case 25: + // DISABLED: improved version in case 30! + // Remove excess created_when from ATStates + // stmt.execute("ALTER TABLE ATStates DROP created_when"); + // stmt.execute("CREATE INDEX ATStateHeightIndex on ATStates (height)"); + break; + + case 26: + // Support for trimming + stmt.execute("ALTER TABLE DatabaseInfo ADD AT_trim_height INT NOT NULL DEFAULT 0"); + stmt.execute("ALTER TABLE DatabaseInfo ADD online_signatures_trim_height INT NOT NULL DEFAULT 0"); + break; + + case 27: + // More indexes + stmt.execute("CREATE INDEX IF NOT EXISTS PaymentTransactionsRecipientIndex ON PaymentTransactions (recipient)"); + stmt.execute("CREATE INDEX IF NOT EXISTS ATTransactionsRecipientIndex ON ATTransactions (recipient)"); + break; + + case 28: + // Latest AT state cache + stmt.execute("CREATE TEMPORARY TABLE IF NOT EXISTS LatestATStates (" + + "AT_address QortalAddress NOT NULL, " + + "height INT NOT NULL" + + ")"); + break; + + case 29: + // Turn off HSQLDB redo-log "blockchain.log" and periodically call "CHECKPOINT" ourselves + stmt.execute("SET FILES LOG FALSE"); + stmt.execute("CHECKPOINT"); + break; + + case 30: { + // Split AT state data off to new table for better performance/management. + + if (!wasPristine && !"mem".equals(HSQLDBRepository.getDbPathname(connection.getMetaData().getURL()))) { + // First, backup node-local data in case user wants to avoid long reshape and use bootstrap instead + try (ResultSet resultSet = stmt.executeQuery("SELECT COUNT(*) FROM MintingAccounts")) { + int rowCount = resultSet.next() ? resultSet.getInt(1) : 0; + if (rowCount > 0) { + stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'MintingAccounts.script'"); + LOGGER.info("Exported sensitive/node-local minting keys into MintingAccounts.script"); + } + } + + try (ResultSet resultSet = stmt.executeQuery("SELECT COUNT(*) FROM TradeBotStates")) { + int rowCount = resultSet.next() ? resultSet.getInt(1) : 0; + if (rowCount > 0) { + stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'TradeBotStates.script'"); + LOGGER.info("Exported sensitive/node-local trade-bot states into TradeBotStates.script"); + } + } + + LOGGER.info("If following reshape takes too long, use bootstrap and import node-local data using API's POST /admin/repository/data"); + } + + // Create new AT-states table without full state data + stmt.execute("CREATE TABLE ATStatesNew (" + + "AT_address QortalAddress, height INTEGER NOT NULL, state_hash ATStateHash NOT NULL, " + + "fees QortalAmount NOT NULL, is_initial BOOLEAN NOT NULL, " + + "PRIMARY KEY (AT_address, height), " + + "FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)"); + stmt.execute("SET TABLE ATStatesNew NEW SPACE"); + stmt.execute("CHECKPOINT"); + + ResultSet resultSet = stmt.executeQuery("SELECT height FROM Blocks ORDER BY height DESC LIMIT 1"); + final int blockchainHeight = resultSet.next() ? resultSet.getInt(1) : 0; + final int heightStep = 100; + + LOGGER.info("Rebuilding AT state summaries in repository - this might take a while... (approx. 2 mins on high-spec)"); + for (int minHeight = 1; minHeight < blockchainHeight; minHeight += heightStep) { + stmt.execute("INSERT INTO ATStatesNew (" + + "SELECT AT_address, height, state_hash, fees, is_initial " + + "FROM ATStates " + + "WHERE height BETWEEN " + minHeight + " AND " + (minHeight + heightStep - 1) + + ")"); + stmt.execute("COMMIT"); + } + stmt.execute("CHECKPOINT"); + + LOGGER.info("Rebuilding AT states height index in repository - this might take about 3x longer..."); + stmt.execute("CREATE INDEX ATStatesHeightIndex ON ATStatesNew (height)"); + stmt.execute("CHECKPOINT"); + + stmt.execute("CREATE TABLE ATStatesData (" + + "AT_address QortalAddress, height INTEGER NOT NULL, state_data ATState NOT NULL, " + + "PRIMARY KEY (height, AT_address), " + + "FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)"); + stmt.execute("SET TABLE ATStatesData NEW SPACE"); + stmt.execute("CHECKPOINT"); + + LOGGER.info("Rebuilding AT state data in repository - this might take a while... (approx. 2 mins on high-spec)"); + for (int minHeight = 1; minHeight < blockchainHeight; minHeight += heightStep) { + stmt.execute("INSERT INTO ATStatesData (" + + "SELECT AT_address, height, state_data " + + "FROM ATstates " + + "WHERE state_data IS NOT NULL " + + "AND height BETWEEN " + minHeight + " AND " + (minHeight + heightStep - 1) + + ")"); + stmt.execute("COMMIT"); + } + stmt.execute("CHECKPOINT"); + + stmt.execute("DROP TABLE ATStates"); + stmt.execute("ALTER TABLE ATStatesNew RENAME TO ATStates"); + stmt.execute("CHECKPOINT"); + break; + } + + case 31: + // Fix latest AT state cache which was previous created as TEMPORARY + stmt.execute("DROP TABLE IF EXISTS LatestATStates"); + stmt.execute("CREATE TABLE IF NOT EXISTS LatestATStates (" + + "AT_address QortalAddress NOT NULL, " + + "height INT NOT NULL, PRIMARY KEY (height, AT_address))"); + break; + + case 32: + // Multiple blockchains, ACCTs and trade-bots + stmt.execute("ALTER TABLE TradeBotStates ADD COLUMN acct_name VARCHAR(40) BEFORE trade_state"); + stmt.execute("UPDATE TradeBotStates SET acct_name = 'BitcoinACCTv1' WHERE acct_name IS NULL"); + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN acct_name SET NOT NULL"); + + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN trade_state RENAME TO trade_state_value"); + + stmt.execute("ALTER TABLE TradeBotStates ADD COLUMN trade_state VARCHAR(40) BEFORE trade_state_value"); + // Any existing values will be BitcoinACCTv1 + StringBuilder updateTradeBotStatesSql = new StringBuilder(1024); + updateTradeBotStatesSql.append("UPDATE TradeBotStates SET (trade_state) = (") + .append("SELECT state_name FROM (VALUES ") + .append( + Arrays.stream(BitcoinACCTv1TradeBot.State.values()) + .map(state -> String.format("(%d, '%s')", state.value, state.name())) + .collect(Collectors.joining(", "))) + .append(") AS BitcoinACCTv1States (state_value, state_name) ") + .append("WHERE state_value = trade_state_value)"); + stmt.execute(updateTradeBotStatesSql.toString()); + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN trade_state SET NOT NULL"); + + stmt.execute("ALTER TABLE TradeBotStates ADD COLUMN foreign_blockchain VARCHAR(40) BEFORE trade_foreign_public_key"); + + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN bitcoin_amount RENAME TO foreign_amount"); + + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN xprv58 RENAME TO foreign_key"); + + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN secret SET NULL"); + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN hash_of_secret SET NULL"); + break; + + case 33: + // PRESENCE transactions + stmt.execute("CREATE TABLE IF NOT EXISTS PresenceTransactions (" + + "signature Signature, nonce INT NOT NULL, presence_type INT NOT NULL, " + + "timestamp_signature Signature NOT NULL, " + TRANSACTION_KEYS + ")"); + break; + + case 34: { + // AT sleep-until-message support + LOGGER.info("Altering AT table in repository - this might take a while... (approx. 20 seconds on high-spec)"); + stmt.execute("ALTER TABLE ATs ADD sleep_until_message_timestamp BIGINT"); + + // Create new AT-states table with new column + stmt.execute("CREATE TABLE ATStatesNew (" + + "AT_address QortalAddress, height INTEGER NOT NULL, state_hash ATStateHash NOT NULL, " + + "fees QortalAmount NOT NULL, is_initial BOOLEAN NOT NULL, sleep_until_message_timestamp BIGINT, " + + "PRIMARY KEY (AT_address, height), " + + "FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)"); + stmt.execute("SET TABLE ATStatesNew NEW SPACE"); + stmt.execute("CHECKPOINT"); + + // Add the height index + LOGGER.info("Adding index to AT states table..."); + stmt.execute("CREATE INDEX ATStatesNewHeightIndex ON ATStatesNew (height)"); + stmt.execute("CHECKPOINT"); + + ResultSet resultSet = stmt.executeQuery("SELECT height FROM Blocks ORDER BY height DESC LIMIT 1"); + final int blockchainHeight = resultSet.next() ? resultSet.getInt(1) : 0; + final int heightStep = 100; + + LOGGER.info("Altering AT states table in repository - this might take a while... (approx. 3 mins on high-spec)"); + for (int minHeight = 1; minHeight < blockchainHeight; minHeight += heightStep) { + stmt.execute("INSERT INTO ATStatesNew (" + + "SELECT AT_address, height, state_hash, fees, is_initial, NULL " + + "FROM ATStates " + + "WHERE height BETWEEN " + minHeight + " AND " + (minHeight + heightStep - 1) + + ")"); + stmt.execute("COMMIT"); + + int processed = Math.min(minHeight + heightStep - 1, blockchainHeight); + double percentage = (double)processed / (double)blockchainHeight * 100.0f; + LOGGER.info(String.format("Processed %d of %d blocks (%.1f%%)", processed, blockchainHeight, percentage)); + } + stmt.execute("CHECKPOINT"); + + stmt.execute("DROP TABLE ATStates"); + stmt.execute("ALTER TABLE ATStatesNew RENAME TO ATStates"); + stmt.execute("ALTER INDEX ATStatesNewHeightIndex RENAME TO ATStatesHeightIndex"); + stmt.execute("CHECKPOINT"); + break; + } + case 35: + // Support for pruning + stmt.execute("ALTER TABLE DatabaseInfo ADD AT_prune_height INT NOT NULL DEFAULT 0"); + stmt.execute("ALTER TABLE DatabaseInfo ADD block_prune_height INT NOT NULL DEFAULT 0"); + break; + + case 36: + // Block archive support + stmt.execute("ALTER TABLE DatabaseInfo ADD block_archive_height INT NOT NULL DEFAULT 0"); + + // Block archive (lookup table to map signature to height) + // Actual data is stored in archive files outside of the database + stmt.execute("CREATE TABLE BlockArchive (signature BlockSignature, height INTEGER NOT NULL, " + + "minted_when EpochMillis NOT NULL, minter QortalPublicKey NOT NULL, " + + "PRIMARY KEY (signature))"); + // For finding blocks by height. + stmt.execute("CREATE INDEX BlockArchiveHeightIndex ON BlockArchive (height)"); + // For finding blocks by the account that minted them. + stmt.execute("CREATE INDEX BlockArchiveMinterIndex ON BlockArchive (minter)"); + // For finding blocks by timestamp or finding height of latest block immediately before timestamp, etc. + stmt.execute("CREATE INDEX BlockArchiveTimestampHeightIndex ON BlockArchive (minted_when, height)"); + // Use a separate table space as this table will be very large. + stmt.execute("SET TABLE BlockArchive NEW SPACE"); + break; + + case 37: + // ARBITRARY transaction updates for off-chain data storage + + // We may want to use a nonce rather than a transaction fee on the data chain + stmt.execute("ALTER TABLE ArbitraryTransactions ADD nonce INT NOT NULL DEFAULT 0"); + // We need to know the total size of the data file(s) associated with each transaction + stmt.execute("ALTER TABLE ArbitraryTransactions ADD size INT NOT NULL DEFAULT 0"); + // Larger data files need to be split into chunks, for easier transmission and greater decentralization + // We store their hashes (and possibly other things) in a metadata file + stmt.execute("ALTER TABLE ArbitraryTransactions ADD metadata_hash VARBINARY(32)"); + // For finding transactions by file hash + stmt.execute("CREATE INDEX ArbitraryDataIndex ON ArbitraryTransactions (is_data_raw, data)"); + break; + + case 38: + // We need the ability for arbitrary transactions to be associated with a name + stmt.execute("ALTER TABLE ArbitraryTransactions ADD name RegisteredName"); + // A "method" specifies how the data should be applied (e.g. PUT or PATCH) + stmt.execute("ALTER TABLE ArbitraryTransactions ADD update_method INTEGER NOT NULL DEFAULT 0"); + // For public data, the AES shared secret needs to be available. This is more for data obfuscation as apposed to actual encryption. + stmt.execute("ALTER TABLE ArbitraryTransactions ADD secret VARBINARY(32)"); + // We want to support compressed and uncompressed data, as well as different compression algorithms + stmt.execute("ALTER TABLE ArbitraryTransactions ADD compression INTEGER NOT NULL DEFAULT 0"); + // An optional identifier string can be used to allow more than one resource per user/service combo + stmt.execute("ALTER TABLE ArbitraryTransactions ADD identifier VARCHAR(64)"); + // For finding transactions by registered name + stmt.execute("CREATE INDEX ArbitraryNameIndex ON ArbitraryTransactions (name)"); + break; + + case 39: + // Add DHT-style lookup table to track file locations + // This maps ARBITRARY transactions to peer addresses, but also includes additional metadata to + // track the local success rate and reachability. It is keyed by a "hash" column, to keep it + // generic, as this way we aren't limited to transaction signatures only. + // Multiple rows with the same hash are allowed, to allow for metadata. Longer term it could be + // reshaped to one row per hash if this is too verbose. + // Transaction signatures are hashed to 32 bytes using SHA256. In doing this we lose the ability + // to join against transaction tables, but on balance the space savings seem more important. + stmt.execute("CREATE TABLE ArbitraryPeers (hash VARBINARY(32) NOT NULL, " + + "peer_address VARCHAR(255), successes INTEGER NOT NULL, failures INTEGER NOT NULL, " + + "last_attempted EpochMillis NOT NULL, last_retrieved EpochMillis NOT NULL, " + + "PRIMARY KEY (hash, peer_address))"); + + // For finding peers by data hash + stmt.execute("CREATE INDEX ArbitraryPeersHashIndex ON ArbitraryPeers (hash)"); + break; + + case 40: + // For looking up name registration transactions based on name or reduced name + stmt.execute("CREATE INDEX RegisterNameNameIndex ON RegisterNameTransactions (name)"); + stmt.execute("CREATE INDEX RegisterNameReducedNameIndex ON RegisterNameTransactions (reduced_name)"); + // For looking up update name transactions based on name, new name, or new reduced name + stmt.execute("CREATE INDEX UpdateNameNameIndex ON UpdateNameTransactions (name)"); + stmt.execute("CREATE INDEX UpdateNameNewNameIndex ON UpdateNameTransactions (new_name)"); + stmt.execute("CREATE INDEX UpdateNameReducedNewNameIndex ON UpdateNameTransactions (reduced_new_name)"); + // For looking up buy name transactions based on name + stmt.execute("CREATE INDEX BuyNameNameIndex ON BuyNameTransactions (name)"); + // For looking up sell name transactions based on name + stmt.execute("CREATE INDEX SellNameNameIndex ON SellNameTransactions (name)"); + break; + default: // nothing to do return false; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBImportExport.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBImportExport.java new file mode 100644 index 000000000..3e6dd5346 --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBImportExport.java @@ -0,0 +1,315 @@ +package org.qortal.repository.hsqldb; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.qortal.data.account.MintingAccountData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.repository.Bootstrap; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.settings.Settings; +import org.qortal.utils.Base58; +import org.qortal.utils.Triple; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Iterator; +import java.util.List; + +public class HSQLDBImportExport { + + private static final Logger LOGGER = LogManager.getLogger(Bootstrap.class); + + public static void backupTradeBotStates(Repository repository, List additional) throws DataException { + HSQLDBImportExport.backupCurrentTradeBotStates(repository, additional); + HSQLDBImportExport.backupArchivedTradeBotStates(repository, additional); + + LOGGER.info("Exported sensitive/node-local data: trade bot states"); + } + + public static void backupMintingAccounts(Repository repository) throws DataException { + HSQLDBImportExport.backupCurrentMintingAccounts(repository); + + LOGGER.info("Exported sensitive/node-local data: minting accounts"); + } + + + /* Trade bot states */ + + /** + * Backs up the trade bot states currently in the repository, without combining them with past ones + * @param repository + * @param additional - any optional extra trade bot states to include in the backup + * @throws DataException + */ + private static void backupCurrentTradeBotStates(Repository repository, List additional) throws DataException { + try { + Path backupDirectory = HSQLDBImportExport.getExportDirectory(true); + + // Load current trade bot data + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + + + // Add any additional entries if specified + if (additional != null && !additional.isEmpty()) { + allTradeBotData.addAll(additional); + } + + // Convert them to JSON objects + JSONArray currentTradeBotDataJson = new JSONArray(); + for (TradeBotData tradeBotData : allTradeBotData) { + JSONObject tradeBotDataJson = tradeBotData.toJson(); + currentTradeBotDataJson.put(tradeBotDataJson); + } + + // Wrap current trade bot data in an object to indicate the type + JSONObject currentTradeBotDataJsonWrapper = new JSONObject(); + currentTradeBotDataJsonWrapper.put("type", "tradeBotStates"); + currentTradeBotDataJsonWrapper.put("dataset", "current"); + currentTradeBotDataJsonWrapper.put("data", currentTradeBotDataJson); + + // Write current trade bot data (just the ones currently in the database) + String fileName = Paths.get(backupDirectory.toString(), "TradeBotStates.json").toString(); + FileWriter writer = new FileWriter(fileName); + writer.write(currentTradeBotDataJsonWrapper.toString(2)); + writer.close(); + + } catch (DataException | IOException e) { + throw new DataException("Unable to export trade bot states from repository"); + } + } + + /** + * Backs up the trade bot states currently in the repository to a separate "archive" file, + * making sure to combine them with any unique states already present in the archive. + * @param repository + * @param additional - any optional extra trade bot states to include in the backup + * @throws DataException + */ + private static void backupArchivedTradeBotStates(Repository repository, List additional) throws DataException { + try { + Path backupDirectory = HSQLDBImportExport.getExportDirectory(true); + + // Load current trade bot data + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + + // Add any additional entries if specified + if (additional != null && !additional.isEmpty()) { + allTradeBotData.addAll(additional); + } + + // Convert them to JSON objects + JSONArray allTradeBotDataJson = new JSONArray(); + for (TradeBotData tradeBotData : allTradeBotData) { + JSONObject tradeBotDataJson = tradeBotData.toJson(); + allTradeBotDataJson.put(tradeBotDataJson); + } + + // We need to combine existing archived TradeBotStates data before overwriting + String fileName = Paths.get(backupDirectory.toString(), "TradeBotStatesArchive.json").toString(); + File tradeBotStatesBackupFile = new File(fileName); + if (tradeBotStatesBackupFile.exists()) { + + String jsonString = new String(Files.readAllBytes(Paths.get(fileName))); + Triple parsedJSON = HSQLDBImportExport.parseJSONString(jsonString); + if (parsedJSON.getA() == null || parsedJSON.getC() == null) { + throw new DataException("Missing data when exporting archived trade bot states"); + } + String type = parsedJSON.getA(); + String dataset = parsedJSON.getB(); + JSONArray data = parsedJSON.getC(); + + if (!type.equals("tradeBotStates") || !dataset.equals("archive")) { + throw new DataException("Format mismatch when exporting archived trade bot states"); + } + + Iterator iterator = data.iterator(); + while(iterator.hasNext()) { + JSONObject existingTradeBotDataItem = (JSONObject)iterator.next(); + String existingTradePrivateKey = (String) existingTradeBotDataItem.get("tradePrivateKey"); + // Check if we already have an entry for this trade + boolean found = allTradeBotData.stream().anyMatch(tradeBotData -> Base58.encode(tradeBotData.getTradePrivateKey()).equals(existingTradePrivateKey)); + if (found == false) + // Add the data from the backup file to our "allTradeBotDataJson" array as it's not currently in the db + allTradeBotDataJson.put(existingTradeBotDataItem); + } + } + + // Wrap all trade bot data in an object to indicate the type + JSONObject allTradeBotDataJsonWrapper = new JSONObject(); + allTradeBotDataJsonWrapper.put("type", "tradeBotStates"); + allTradeBotDataJsonWrapper.put("dataset", "archive"); + allTradeBotDataJsonWrapper.put("data", allTradeBotDataJson); + + // Write ALL trade bot data to archive (current plus states that are no longer in the database) + FileWriter writer = new FileWriter(fileName); + writer.write(allTradeBotDataJsonWrapper.toString(2)); + writer.close(); + + } catch (DataException | IOException e) { + throw new DataException("Unable to export trade bot states from repository"); + } + } + + + /* Minting accounts */ + + /** + * Backs up the minting accounts currently in the repository, without combining them with past ones + * @param repository + * @throws DataException + */ + private static void backupCurrentMintingAccounts(Repository repository) throws DataException { + try { + Path backupDirectory = HSQLDBImportExport.getExportDirectory(true); + + // Load current trade bot data + List allMintingAccountData = repository.getAccountRepository().getMintingAccounts(); + JSONArray currentMintingAccountJson = new JSONArray(); + for (MintingAccountData mintingAccountData : allMintingAccountData) { + JSONObject mintingAccountDataJson = mintingAccountData.toJson(); + currentMintingAccountJson.put(mintingAccountDataJson); + } + + // Wrap current trade bot data in an object to indicate the type + JSONObject currentMintingAccountDataJsonWrapper = new JSONObject(); + currentMintingAccountDataJsonWrapper.put("type", "mintingAccounts"); + currentMintingAccountDataJsonWrapper.put("dataset", "current"); + currentMintingAccountDataJsonWrapper.put("data", currentMintingAccountJson); + + // Write current trade bot data (just the ones currently in the database) + String fileName = Paths.get(backupDirectory.toString(), "MintingAccounts.json").toString(); + FileWriter writer = new FileWriter(fileName); + writer.write(currentMintingAccountDataJsonWrapper.toString(2)); + writer.close(); + + } catch (DataException | IOException e) { + throw new DataException("Unable to export minting accounts from repository"); + } + } + + + /* Utils */ + + /** + * Imports data from supplied file + * Data type is loaded from the file itself, and if missing, TradeBotStates is assumed + * + * @param filename + * @param repository + * @throws DataException + * @throws IOException + */ + public static void importDataFromFile(String filename, Repository repository) throws DataException, IOException { + Path path = Paths.get(filename); + if (!path.toFile().exists()) { + throw new FileNotFoundException(String.format("File doesn't exist: %s", filename)); + } + byte[] fileContents = Files.readAllBytes(path); + if (fileContents == null) { + throw new FileNotFoundException(String.format("Unable to read file contents: %s", filename)); + } + + LOGGER.info(String.format("Importing %s into repository ...", filename)); + + String jsonString = new String(fileContents); + Triple parsedJSON = HSQLDBImportExport.parseJSONString(jsonString); + if (parsedJSON.getA() == null || parsedJSON.getC() == null) { + throw new DataException(String.format("Missing data when importing %s into repository", filename)); + } + String type = parsedJSON.getA(); + JSONArray data = parsedJSON.getC(); + + Iterator iterator = data.iterator(); + while(iterator.hasNext()) { + JSONObject dataJsonObject = (JSONObject)iterator.next(); + + if (type.equals("tradeBotStates")) { + HSQLDBImportExport.importTradeBotDataJSON(dataJsonObject, repository); + } + else if (type.equals("mintingAccounts")) { + HSQLDBImportExport.importMintingAccountDataJSON(dataJsonObject, repository); + } + else { + throw new DataException(String.format("Unrecognized data type when importing %s into repository", filename)); + } + + } + LOGGER.info(String.format("Imported %s into repository from %s", type, filename)); + } + + private static void importTradeBotDataJSON(JSONObject tradeBotDataJson, Repository repository) throws DataException { + TradeBotData tradeBotData = TradeBotData.fromJson(tradeBotDataJson); + repository.getCrossChainRepository().save(tradeBotData); + } + + private static void importMintingAccountDataJSON(JSONObject mintingAccountDataJson, Repository repository) throws DataException { + MintingAccountData mintingAccountData = MintingAccountData.fromJson(mintingAccountDataJson); + repository.getAccountRepository().save(mintingAccountData); + } + + public static Path getExportDirectory(boolean createIfNotExists) throws DataException { + Path backupPath = Paths.get(Settings.getInstance().getExportPath()); + + if (createIfNotExists) { + // Create the qortal-backup folder if it doesn't exist + try { + Files.createDirectories(backupPath); + } catch (IOException e) { + LOGGER.info(String.format("Unable to create %s folder", backupPath.toString())); + throw new DataException(String.format("Unable to create %s folder", backupPath.toString())); + } + } + + return backupPath; + } + + /** + * Parses a JSON string and returns "data", "type", and "dataset" fields. + * In the case of legacy JSON files with no type, they are assumed to be TradeBotStates archives, + * as we had never implemented this for any other types. + * + * @param jsonString + * @return Triple (type, dataset, data) + */ + public static Triple parseJSONString(String jsonString) throws DataException { + String type = null; + String dataset = null; + JSONArray data = null; + + try { + // Firstly try importing the new format + JSONObject jsonData = new JSONObject(jsonString); + if (jsonData != null && jsonData.getString("type") != null) { + + type = jsonData.getString("type"); + dataset = jsonData.getString("dataset"); + data = jsonData.getJSONArray("data"); + } + + } catch (JSONException e) { + // Could be a legacy format which didn't contain a type or any other outer keys, so try importing that + // Treat these as TradeBotStates archives, given that this was the only type previously implemented + try { + type = "tradeBotStates"; + dataset = "archive"; + data = new JSONArray(jsonString); + + } catch (JSONException e2) { + // Still failed, so give up + throw new DataException("Couldn't import JSON file"); + } + } + + return new Triple(type, dataset, data); + } + +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBMessageRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBMessageRepository.java new file mode 100644 index 000000000..f00c79fc3 --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBMessageRepository.java @@ -0,0 +1,85 @@ +package org.qortal.repository.hsqldb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.MessageRepository; +import org.qortal.transaction.Transaction.TransactionType; + +public class HSQLDBMessageRepository implements MessageRepository { + + protected HSQLDBRepository repository; + + public HSQLDBMessageRepository(HSQLDBRepository repository) { + this.repository = repository; + } + + @Override + public List getMessagesByParticipants(byte[] senderPublicKey, + String recipient, Integer limit, Integer offset, Boolean reverse) throws DataException { + if (senderPublicKey == null && recipient == null) + throw new DataException("At least one of senderPublicKey or recipient required to fetch matching messages"); + + StringBuilder sql = new StringBuilder(1024); + sql.append("SELECT signature from MessageTransactions " + + "JOIN Transactions USING (signature) " + + "JOIN BlockTransactions ON transaction_signature = signature " + + "WHERE "); + + List whereClauses = new ArrayList<>(); + List bindParams = new ArrayList<>(); + + if (senderPublicKey != null) { + whereClauses.add("sender = ?"); + bindParams.add(senderPublicKey); + } + + if (recipient != null) { + whereClauses.add("recipient = ?"); + bindParams.add(recipient); + } + + sql.append(String.join(" AND ", whereClauses)); + + sql.append("ORDER BY Transactions.created_when"); + sql.append((reverse == null || !reverse) ? " ASC" : " DESC"); + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List messageTransactionsData = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return messageTransactionsData; + + do { + byte[] signature = resultSet.getBytes(1); + + TransactionData transactionData = this.repository.getTransactionRepository().fromSignature(signature); + if (transactionData == null || transactionData.getType() != TransactionType.MESSAGE) + throw new DataException("Inconsistent data from repository when fetching message"); + + messageTransactionsData.add((MessageTransactionData) transactionData); + } while (resultSet.next()); + + return messageTransactionsData; + } catch (SQLException e) { + throw new DataException("Unable to fetch matching messages from repository", e); + } + } + + @Override + public boolean exists(byte[] senderPublicKey, String recipient, byte[] messageData) throws DataException { + try { + return this.repository.exists("MessageTransactions", "sender = ? AND recipient = ? AND data = ?", senderPublicKey, recipient, messageData); + } catch (SQLException e) { + throw new DataException("Unable to check for existing message in repository", e); + } + } + +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 9c0ad9abe..61f4b76fb 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -1,5 +1,6 @@ package org.qortal.repository.hsqldb; +import java.awt.TrayIcon.MessageType; import java.io.File; import java.io.IOException; import java.math.BigDecimal; @@ -14,32 +15,19 @@ import java.sql.SQLException; import java.sql.Savepoint; import java.sql.Statement; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Deque; -import java.util.List; +import java.util.*; +import java.util.concurrent.TimeoutException; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.account.PrivateKeyAccount; import org.qortal.crypto.Crypto; -import org.qortal.repository.ATRepository; -import org.qortal.repository.AccountRepository; -import org.qortal.repository.ArbitraryRepository; -import org.qortal.repository.AssetRepository; -import org.qortal.repository.BlockRepository; -import org.qortal.repository.ChatRepository; -import org.qortal.repository.DataException; -import org.qortal.repository.GroupRepository; -import org.qortal.repository.NameRepository; -import org.qortal.repository.NetworkRepository; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.repository.TransactionRepository; -import org.qortal.repository.VotingRepository; +import org.qortal.globalization.Translator; +import org.qortal.gui.SysTray; +import org.qortal.repository.*; import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository; import org.qortal.settings.Settings; @@ -47,19 +35,42 @@ public class HSQLDBRepository implements Repository { private static final Logger LOGGER = LogManager.getLogger(HSQLDBRepository.class); + public static final Object CHECKPOINT_LOCK = new Object(); + + // "serialization failure" + private static final Integer DEADLOCK_ERROR_CODE = Integer.valueOf(-4861); + protected Connection connection; - protected Deque savepoints; + protected final Deque savepoints = new ArrayDeque<>(3); protected boolean debugState = false; protected Long slowQueryThreshold = null; protected List sqlStatements; protected long sessionId; + protected final Map preparedStatementCache = new HashMap<>(); + // We want the same object corresponding to the actual DB + protected final Object trimHeightsLock = RepositoryManager.getRepositoryFactory(); + protected final Object latestATStatesLock = RepositoryManager.getRepositoryFactory(); + + private final ATRepository atRepository = new HSQLDBATRepository(this); + private final AccountRepository accountRepository = new HSQLDBAccountRepository(this); + private final ArbitraryRepository arbitraryRepository = new HSQLDBArbitraryRepository(this); + private final AssetRepository assetRepository = new HSQLDBAssetRepository(this); + private final BlockRepository blockRepository = new HSQLDBBlockRepository(this); + private final BlockArchiveRepository blockArchiveRepository = new HSQLDBBlockArchiveRepository(this); + private final ChatRepository chatRepository = new HSQLDBChatRepository(this); + private final CrossChainRepository crossChainRepository = new HSQLDBCrossChainRepository(this); + private final GroupRepository groupRepository = new HSQLDBGroupRepository(this); + private final MessageRepository messageRepository = new HSQLDBMessageRepository(this); + private final NameRepository nameRepository = new HSQLDBNameRepository(this); + private final NetworkRepository networkRepository = new HSQLDBNetworkRepository(this); + private final TransactionRepository transactionRepository = new HSQLDBTransactionRepository(this); + private final VotingRepository votingRepository = new HSQLDBVotingRepository(this); // Constructors // NB: no visibility modifier so only callable from within same package /* package */ HSQLDBRepository(Connection connection) throws DataException { this.connection = connection; - this.savepoints = new ArrayDeque<>(3); this.slowQueryThreshold = Settings.getInstance().getSlowQueryThreshold(); if (this.slowQueryThreshold != null) @@ -80,64 +91,82 @@ public class HSQLDBRepository implements Repository { throw new DataException("Unable to fetch session ID from repository", e); } - assertEmptyTransaction("connection creation"); + // synchronize to block new connections if checkpointing in progress + synchronized (CHECKPOINT_LOCK) { + assertEmptyTransaction("connection creation"); + } } // Getters / setters @Override public ATRepository getATRepository() { - return new HSQLDBATRepository(this); + return this.atRepository; } @Override public AccountRepository getAccountRepository() { - return new HSQLDBAccountRepository(this); + return this.accountRepository; } @Override public ArbitraryRepository getArbitraryRepository() { - return new HSQLDBArbitraryRepository(this); + return this.arbitraryRepository; } @Override public AssetRepository getAssetRepository() { - return new HSQLDBAssetRepository(this); + return this.assetRepository; } @Override public BlockRepository getBlockRepository() { - return new HSQLDBBlockRepository(this); + return this.blockRepository; + } + + @Override + public BlockArchiveRepository getBlockArchiveRepository() { + return this.blockArchiveRepository; } @Override public ChatRepository getChatRepository() { - return new HSQLDBChatRepository(this); + return this.chatRepository; + } + + @Override + public CrossChainRepository getCrossChainRepository() { + return this.crossChainRepository; } @Override public GroupRepository getGroupRepository() { - return new HSQLDBGroupRepository(this); + return this.groupRepository; + } + + @Override + public MessageRepository getMessageRepository() { + return this.messageRepository; } @Override public NameRepository getNameRepository() { - return new HSQLDBNameRepository(this); + return this.nameRepository; } @Override public NetworkRepository getNetworkRepository() { - return new HSQLDBNetworkRepository(this); + return this.networkRepository; } @Override public TransactionRepository getTransactionRepository() { - return new HSQLDBTransactionRepository(this); + return this.transactionRepository; } @Override public VotingRepository getVotingRepository() { - return new HSQLDBVotingRepository(this); + return this.votingRepository; } @Override @@ -154,8 +183,20 @@ public void setDebug(boolean debugState) { @Override public void saveChanges() throws DataException { + long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis(); + try { this.connection.commit(); + + if (this.slowQueryThreshold != null) { + long queryTime = System.currentTimeMillis() - beforeQuery; + + if (queryTime > this.slowQueryThreshold) { + LOGGER.info(() -> String.format("[Session %d] HSQLDB COMMIT took %d ms", this.sessionId, queryTime), new SQLException("slow commit")); + + logStatements(); + } + } } catch (SQLException e) { throw new DataException("commit error", e); } finally { @@ -179,7 +220,7 @@ public void discardChanges() throws DataException { this.savepoints.clear(); // Before clearing statements so we can log what led to assertion error - assertEmptyTransaction("transaction commit"); + assertEmptyTransaction("transaction rollback"); if (this.sqlStatements != null) this.sqlStatements.clear(); @@ -227,14 +268,22 @@ public void rollbackToSavepoint() throws DataException { public void close() throws DataException { // Already closed? No need to do anything but maybe report double-call if (this.connection == null) { - LOGGER.warn("HSQLDBRepository.close() called when repository already closed", new Exception("Repository already closed")); + LOGGER.warn("HSQLDBRepository.close() called when repository already closed. This is expected when bootstrapping."); return; } - try (Statement stmt = this.connection.createStatement()) { + try { assertEmptyTransaction("connection close"); - // give connection back to the pool + // Assume we are not going to be GC'd for a while + this.preparedStatementCache.clear(); + this.sqlStatements = null; + this.savepoints.clear(); + + // If a checkpoint has been requested, we could perform that now + this.maybeCheckpoint(); + + // Give connection back to the pool this.connection.close(); this.connection = null; } catch (SQLException e) { @@ -242,6 +291,58 @@ public void close() throws DataException { } } + private void maybeCheckpoint() throws DataException { + // To serialize checkpointing and to block new sessions when checkpointing in progress + synchronized (CHECKPOINT_LOCK) { + Boolean quickCheckpointRequest = RepositoryManager.getRequestedCheckpoint(); + if (quickCheckpointRequest == null) + return; + + // We can only perform a CHECKPOINT if no other HSQLDB session is mid-transaction, + // otherwise the CHECKPOINT blocks for COMMITs and other threads can't open HSQLDB sessions + // due to HSQLDB blocking until CHECKPOINT finishes - i.e. deadlock + String sql = "SELECT COUNT(*) " + + "FROM Information_schema.system_sessions " + + "WHERE transaction = TRUE"; + + try { + PreparedStatement pstmt = this.cachePreparedStatement(sql); + + if (!pstmt.execute()) + throw new DataException("Unable to check repository session status"); + + try (ResultSet resultSet = pstmt.getResultSet()) { + if (resultSet == null || !resultSet.next()) + // Failed to even find HSQLDB session info! + throw new DataException("No results when checking repository session status"); + + int transactionCount = resultSet.getInt(1); + + if (transactionCount > 0) + // We can't safely perform CHECKPOINT due to ongoing SQL transactions + return; + } + + LOGGER.info("Performing repository CHECKPOINT..."); + + if (Settings.getInstance().getShowCheckpointNotification()) + SysTray.getInstance().showMessage(Translator.INSTANCE.translate("SysTray", "DB_CHECKPOINT"), + Translator.INSTANCE.translate("SysTray", "PERFORMING_DB_CHECKPOINT"), + MessageType.INFO); + + try (Statement stmt = this.connection.createStatement()) { + stmt.execute(Boolean.TRUE.equals(quickCheckpointRequest) ? "CHECKPOINT" : "CHECKPOINT DEFRAG"); + } + + // Completed! + LOGGER.info("Repository CHECKPOINT completed!"); + RepositoryManager.setRequestedCheckpoint(null); + } catch (SQLException e) { + throw new DataException("Unable to check repository session status", e); + } + } + } + @Override public void rebuild() throws DataException { LOGGER.info("Rebuilding repository from scratch"); @@ -264,11 +365,12 @@ public void rebuild() throws DataException { Path oldRepoDirPath = Paths.get(dbPathname).getParent(); // Delete old repository files - Files.walk(oldRepoDirPath) - .sorted(Comparator.reverseOrder()) + try (Stream paths = Files.walk(oldRepoDirPath)) { + paths.sorted(Comparator.reverseOrder()) .map(Path::toFile) .filter(file -> file.getPath().startsWith(dbPathname)) .forEach(File::delete); + } } } catch (NoSuchFileException e) { // Nothing to remove @@ -278,57 +380,113 @@ public void rebuild() throws DataException { } @Override - public void backup(boolean quick) throws DataException { - if (!quick) - // First perform a CHECKPOINT + public void backup(boolean quick, String name, Long timeout) throws DataException, TimeoutException { + synchronized (CHECKPOINT_LOCK) { + + // We can only perform a CHECKPOINT if no other HSQLDB session is mid-transaction, + // otherwise the CHECKPOINT blocks for COMMITs and other threads can't open HSQLDB sessions + // due to HSQLDB blocking until CHECKPOINT finishes - i.e. deadlock. + // Since we don't want to give up too easily, it's best to wait until the other transaction + // count reaches zero, and then continue. + this.blockUntilNoOtherTransactions(timeout); + + if (!quick) + // First perform a CHECKPOINT + try (Statement stmt = this.connection.createStatement()) { + LOGGER.info("Performing maintenance - this will take a while..."); + stmt.execute("CHECKPOINT"); + stmt.execute("CHECKPOINT DEFRAG"); + LOGGER.info("Maintenance completed"); + } catch (SQLException e) { + throw new DataException("Unable to prepare repository for backup"); + } + + // Clean out any previous backup + try { + String connectionUrl = this.connection.getMetaData().getURL(); + String dbPathname = getDbPathname(connectionUrl); + if (dbPathname == null) + throw new DataException("Unable to locate repository for backup?"); + + // Doesn't really make sense to backup an in-memory database... + if (dbPathname.equals("mem")) { + LOGGER.debug("Ignoring request to backup in-memory repository!"); + return; + } + + String backupUrl = buildBackupUrl(dbPathname, name); + String backupPathname = getDbPathname(backupUrl); + if (backupPathname == null) + throw new DataException("Unable to determine location for repository backup?"); + + Path backupDirPath = Paths.get(backupPathname).getParent(); + String backupDirPathname = backupDirPath.toString(); + + try (Stream paths = Files.walk(backupDirPath)) { + paths.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .filter(file -> file.getPath().startsWith(backupDirPathname)) + .forEach(File::delete); + } + } catch (NoSuchFileException e) { + // Nothing to remove + } catch (SQLException | IOException e) { + throw new DataException("Unable to remove previous repository backup"); + } + + // Actually create backup try (Statement stmt = this.connection.createStatement()) { - stmt.execute("CHECKPOINT DEFRAG"); + LOGGER.info("Backing up repository..."); + stmt.execute(String.format("BACKUP DATABASE TO '%s/' BLOCKING AS FILES", name)); + LOGGER.info("Backup completed"); } catch (SQLException e) { - throw new DataException("Unable to prepare repository for backup"); + throw new DataException("Unable to backup repository"); } - // Clean out any previous backup - try { - String connectionUrl = this.connection.getMetaData().getURL(); - String dbPathname = getDbPathname(connectionUrl); - if (dbPathname == null) - throw new DataException("Unable to locate repository for backup?"); - - // Doesn't really make sense to backup an in-memory database... - if (dbPathname.equals("mem")) { - LOGGER.debug("Ignoring request to backup in-memory repository!"); - return; - } + } + } - String backupUrl = buildBackupUrl(dbPathname); - String backupPathname = getDbPathname(backupUrl); - if (backupPathname == null) - throw new DataException("Unable to determine location for repository backup?"); + @Override + public void performPeriodicMaintenance(Long timeout) throws DataException, TimeoutException { + synchronized (CHECKPOINT_LOCK) { - Path backupDirPath = Paths.get(backupPathname).getParent(); - String backupDirPathname = backupDirPath.toString(); + // We can only perform a CHECKPOINT if no other HSQLDB session is mid-transaction, + // otherwise the CHECKPOINT blocks for COMMITs and other threads can't open HSQLDB sessions + // due to HSQLDB blocking until CHECKPOINT finishes - i.e. deadlock. + // Since we don't want to give up too easily, it's best to wait until the other transaction + // count reaches zero, and then continue. + this.blockUntilNoOtherTransactions(timeout); - Files.walk(backupDirPath) - .sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .filter(file -> file.getPath().startsWith(backupDirPathname)) - .forEach(File::delete); - } catch (NoSuchFileException e) { - // Nothing to remove - } catch (SQLException | IOException e) { - throw new DataException("Unable to remove previous repository backup"); + // Defrag DB - takes a while! + try (Statement stmt = this.connection.createStatement()) { + LOGGER.info("performing maintenance - this will take a while"); + stmt.execute("CHECKPOINT"); + stmt.execute("CHECKPOINT DEFRAG"); + LOGGER.info("maintenance completed"); + } catch (SQLException e) { + throw new DataException("Unable to defrag repository"); + } } + } - // Actually create backup - try (Statement stmt = this.connection.createStatement()) { - stmt.execute("BACKUP DATABASE TO 'backup/' NOT BLOCKING AS FILES"); - } catch (SQLException e) { - throw new DataException("Unable to backup repository"); - } + @Override + public void exportNodeLocalData() throws DataException { + HSQLDBImportExport.backupTradeBotStates(this, null); + HSQLDBImportExport.backupMintingAccounts(this); + } + + @Override + public void importDataFromFile(String filename) throws DataException, IOException { + HSQLDBImportExport.importDataFromFile(filename, this); + } + + @Override + public void checkConsistency() throws DataException { + this.getATRepository().checkConsistency(); } /** Returns DB pathname from passed connection URL. If memory DB, returns "mem". */ - private static String getDbPathname(String connectionUrl) { + /*package*/ static String getDbPathname(String connectionUrl) { Pattern pattern = Pattern.compile("hsqldb:(mem|file):(.*?)(;|$)"); Matcher matcher = pattern.matcher(connectionUrl); @@ -341,22 +499,22 @@ private static String getDbPathname(String connectionUrl) { return matcher.group(2); } - private static String buildBackupUrl(String dbPathname) { + private static String buildBackupUrl(String dbPathname, String backupName) { Path oldRepoPath = Paths.get(dbPathname); Path oldRepoDirPath = oldRepoPath.getParent(); Path oldRepoFilePath = oldRepoPath.getFileName(); // Try to open backup. We need to remove "create=true" and insert "backup" dir before final filename. - String backupUrlTemplate = "jdbc:hsqldb:file:%s%sbackup%s%s;create=false;hsqldb.full_log_replay=true"; - return String.format(backupUrlTemplate, oldRepoDirPath.toString(), File.separator, File.separator, oldRepoFilePath.toString()); + String backupUrlTemplate = "jdbc:hsqldb:file:%s%s%s%s%s;create=false;hsqldb.full_log_replay=true"; + return String.format(backupUrlTemplate, oldRepoDirPath.toString(), File.separator, backupName, File.separator, oldRepoFilePath.toString()); } - /* package */ static void attemptRecovery(String connectionUrl) throws DataException { + /* package */ static void attemptRecovery(String connectionUrl, String name) throws DataException { String dbPathname = getDbPathname(connectionUrl); if (dbPathname == null) throw new DataException("Unable to locate repository for backup?"); - String backupUrl = buildBackupUrl(dbPathname); + String backupUrl = buildBackupUrl(dbPathname, name); Path oldRepoDirPath = Paths.get(dbPathname).getParent(); // Attempt connection to backup to see if it is viable @@ -364,11 +522,12 @@ private static String buildBackupUrl(String dbPathname) { LOGGER.info("Attempting repository recovery using backup"); // Move old repository files out the way - Files.walk(oldRepoDirPath) - .sorted(Comparator.reverseOrder()) + try (Stream paths = Files.walk(oldRepoDirPath)) { + paths.sorted(Comparator.reverseOrder()) .map(Path::toFile) .filter(file -> file.getPath().startsWith(dbPathname)) .forEach(File::delete); + } try (Statement stmt = connection.createStatement()) { // Now "backup" the backup back to original repository location (the parent). @@ -408,7 +567,33 @@ public PreparedStatement prepareStatement(String sql) throws SQLException { if (this.sqlStatements != null) this.sqlStatements.add(sql); - return this.connection.prepareStatement(sql); + return cachePreparedStatement(sql); + } + + private PreparedStatement cachePreparedStatement(String sql) throws SQLException { + /* + * We cache a duplicate PreparedStatement for this SQL string, + * which we never close, which means HSQLDB also caches a parsed, + * prepared statement that can be reused for subsequent + * calls to HSQLDB.prepareStatement(sql). + * + * See org.hsqldb.StatementManager for more details. + */ + PreparedStatement preparedStatement = this.preparedStatementCache.get(sql); + if (preparedStatement == null || preparedStatement.isClosed()) { + if (preparedStatement != null) + // This shouldn't occur, so log, but recompile + LOGGER.debug(() -> String.format("Recompiling closed PreparedStatement: %s", sql)); + + preparedStatement = this.connection.prepareStatement(sql); + this.preparedStatementCache.put(sql, preparedStatement); + } else { + // Clean up ready for reuse + preparedStatement.clearBatch(); + preparedStatement.clearParameters(); + } + + return preparedStatement; } /** @@ -424,9 +609,8 @@ public PreparedStatement prepareStatement(String sql) throws SQLException { public ResultSet checkedExecute(String sql, Object... objects) throws SQLException { PreparedStatement preparedStatement = this.prepareStatement(sql); - // Close the PreparedStatement when the ResultSet is closed otherwise there's a potential resource leak. - // We can't use try-with-resources here as closing the PreparedStatement on return would also prematurely close the ResultSet. - preparedStatement.closeOnCompletion(); + // We don't close the PreparedStatement when the ResultSet is closed because we cached PreparedStatements now. + // They are cleaned up when connection/session is closed. long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis(); @@ -436,7 +620,7 @@ public ResultSet checkedExecute(String sql, Object... objects) throws SQLExcepti long queryTime = System.currentTimeMillis() - beforeQuery; if (queryTime > this.slowQueryThreshold) { - LOGGER.info(() -> String.format("HSQLDB query took %d ms: %s", queryTime, sql), new SQLException("slow query")); + LOGGER.info(() -> String.format("[Session %d] HSQLDB query took %d ms: %s", this.sessionId, queryTime, sql), new SQLException("slow query")); logStatements(); } @@ -454,7 +638,7 @@ public ResultSet checkedExecute(String sql, Object... objects) throws SQLExcepti * @param objects * @throws SQLException */ - private void prepareExecute(PreparedStatement preparedStatement, Object... objects) throws SQLException { + private void bindStatementParams(PreparedStatement preparedStatement, Object... objects) throws SQLException { for (int i = 0; i < objects.length; ++i) // Special treatment for BigDecimals so that they retain their "scale", // which would otherwise be assumed as 0. @@ -475,10 +659,13 @@ private void prepareExecute(PreparedStatement preparedStatement, Object... objec * @throws SQLException */ private ResultSet checkedExecuteResultSet(PreparedStatement preparedStatement, Object... objects) throws SQLException { - prepareExecute(preparedStatement, objects); + bindStatementParams(preparedStatement, objects); - if (!preparedStatement.execute()) - throw new SQLException("Fetching from database produced no results"); + // synchronize to block new executions if checkpointing in progress + synchronized (CHECKPOINT_LOCK) { + if (!preparedStatement.execute()) + throw new SQLException("Fetching from database produced no results"); + } ResultSet resultSet = preparedStatement.getResultSet(); if (resultSet == null) @@ -493,36 +680,66 @@ private ResultSet checkedExecuteResultSet(PreparedStatement preparedStatement, O /** * Execute PreparedStatement and return changed row count. * - * @param preparedStatement + * @param sql * @param objects * @return number of changed rows * @throws SQLException */ - /* package */ int checkedExecuteUpdateCount(String sql, Object... objects) throws SQLException { - try (PreparedStatement preparedStatement = this.prepareStatement(sql)) { - prepareExecute(preparedStatement, objects); + /* package */ int executeCheckedUpdate(String sql, Object... objects) throws SQLException { + return this.executeCheckedBatchUpdate(sql, Collections.singletonList(objects)); + } + + /** + * Execute batched PreparedStatement + * + * @param sql + * @param batchedObjects + * @return number of changed rows + * @throws SQLException + */ + /* package */ int executeCheckedBatchUpdate(String sql, List batchedObjects) throws SQLException { + // Nothing to do? + if (batchedObjects == null || batchedObjects.isEmpty()) + return 0; - long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis(); + PreparedStatement preparedStatement = this.prepareStatement(sql); + for (Object[] objects : batchedObjects) { + this.bindStatementParams(preparedStatement, objects); + preparedStatement.addBatch(); + } - if (preparedStatement.execute()) - throw new SQLException("Database produced results, not row count"); + long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis(); - if (this.slowQueryThreshold != null) { - long queryTime = System.currentTimeMillis() - beforeQuery; + int[] updateCounts = null; + try { + updateCounts = preparedStatement.executeBatch(); + } catch (SQLException e) { + if (isDeadlockException(e)) + // We want more info on what other DB sessions are doing to cause this + examineException(e); - if (queryTime > this.slowQueryThreshold) { - LOGGER.info(() -> String.format("HSQLDB query took %d ms: %s", queryTime, sql), new SQLException("slow query")); + throw e; + } - logStatements(); - } + if (this.slowQueryThreshold != null) { + long queryTime = System.currentTimeMillis() - beforeQuery; + + if (queryTime > this.slowQueryThreshold) { + LOGGER.info(() -> String.format("[Session %d] HSQLDB query took %d ms: %s", this.sessionId, queryTime, sql), new SQLException("slow query")); + + logStatements(); } + } - int rowCount = preparedStatement.getUpdateCount(); - if (rowCount == -1) + int totalCount = 0; + for (int i = 0; i < updateCounts.length; ++i) { + if (updateCounts[i] < 0) throw new SQLException("Database returned invalid row count"); - return rowCount; + totalCount += updateCounts[i]; } + + return totalCount; } /** @@ -592,7 +809,25 @@ public int delete(String tableName, String whereClause, Object... objects) throw sql.append(" WHERE "); sql.append(whereClause); - return this.checkedExecuteUpdateCount(sql.toString(), objects); + return this.executeCheckedUpdate(sql.toString(), objects); + } + + /** + * Delete rows from database table. + * + * @param tableName + * @param whereClause + * @param batchedObjects + * @throws SQLException + */ + public int deleteBatch(String tableName, String whereClause, List batchedObjects) throws SQLException { + StringBuilder sql = new StringBuilder(256); + sql.append("DELETE FROM "); + sql.append(tableName); + sql.append(" WHERE "); + sql.append(whereClause); + + return this.executeCheckedBatchUpdate(sql.toString(), batchedObjects); } /** @@ -606,7 +841,7 @@ public int delete(String tableName) throws SQLException { sql.append("DELETE FROM "); sql.append(tableName); - return this.checkedExecuteUpdateCount(sql.toString()); + return this.executeCheckedUpdate(sql.toString()); } /** @@ -656,15 +891,18 @@ public static void limitOffsetSql(StringBuilder stringBuilder, Integer limit, In *

* (Convenience method for HSQLDB repository subclasses). */ - /* package */ static void temporaryValuesTableSql(StringBuilder stringBuilder, List values, String tableName, String columnName) { + /* package */ static void temporaryValuesTableSql(StringBuilder stringBuilder, Collection values, String tableName, String columnName) { stringBuilder.append("(VALUES "); - for (int i = 0; i < values.size(); ++i) { - if (i != 0) + boolean first = true; + for (Object value : values) { + if (first) + first = false; + else stringBuilder.append(", "); stringBuilder.append("("); - stringBuilder.append(values.get(i)); + stringBuilder.append(value); stringBuilder.append(")"); } @@ -684,15 +922,17 @@ public void logStatements() { if (this.sqlStatements == null) return; - LOGGER.info(() -> String.format("HSQLDB SQL statements (session %d) leading up to this were:", this.sessionId)); + LOGGER.info(() -> String.format("[Session %d] HSQLDB SQL statements leading up to this were:", this.sessionId)); for (String sql : this.sqlStatements) - LOGGER.info(sql); + LOGGER.info(() -> String.format("[Session %d] %s", this.sessionId, sql)); } - /** Logs other HSQLDB sessions then re-throws passed exception */ - public SQLException examineException(SQLException e) throws SQLException { - LOGGER.error(String.format("HSQLDB error (session %d): %s", this.sessionId, e.getMessage()), e); + /** Logs other HSQLDB sessions then returns passed exception */ + public SQLException examineException(SQLException e) { + // TODO: could log at DEBUG for deadlocks by checking RepositoryManager.isDeadlockRelated(e)? + + LOGGER.error(() -> String.format("[Session %d] HSQLDB error: %s", this.sessionId, e.getMessage()), e); logStatements(); @@ -710,7 +950,11 @@ public SQLException examineException(SQLException e) throws SQLException { String thisWaitingFor = resultSet.getString(5); String currentStatement = resultSet.getString(6); - LOGGER.error(String.format("Session %d, %s transaction (size %d), waiting for this '%s', this waiting for '%s', current statement: %s", + // Skip logging idle sessions + if (transactionSize == 0 && waitingForThis.isEmpty() && thisWaitingFor.isEmpty() && currentStatement.isEmpty()) + continue; + + LOGGER.error(() -> String.format("Session %d, %s transaction (size %d), waiting for this '%s', this waiting for '%s', current statement: %s", systemSessionId, (inTransaction ? "in" : "not in"), transactionSize, waitingForThis, thisWaitingFor, currentStatement)); } while (resultSet.next()); } catch (SQLException de) { @@ -722,14 +966,19 @@ public SQLException examineException(SQLException e) throws SQLException { } private void assertEmptyTransaction(String context) throws DataException { - try (Statement stmt = this.connection.createStatement()) { + String sql = "SELECT transaction, transaction_size FROM information_schema.system_sessions WHERE session_id = ?"; + + try { + PreparedStatement stmt = this.cachePreparedStatement(sql); + stmt.setLong(1, this.sessionId); + // Diagnostic check for uncommitted changes - if (!stmt.execute("SELECT transaction, transaction_size FROM information_schema.system_sessions WHERE session_id = " + this.sessionId)) // TRANSACTION_SIZE() broken? + if (!stmt.execute()) // TRANSACTION_SIZE() broken? throw new DataException("Unable to check repository status after " + context); try (ResultSet resultSet = stmt.getResultSet()) { if (resultSet == null || !resultSet.next()) { - LOGGER.warn(String.format("Unable to check repository status after %s", context)); + LOGGER.warn(() -> String.format("Unable to check repository status after %s", context)); return; } @@ -737,7 +986,11 @@ private void assertEmptyTransaction(String context) throws DataException { int transactionCount = resultSet.getInt(2); if (inTransaction && transactionCount != 0) { - LOGGER.warn(String.format("Uncommitted changes (%d) after %s, session [%d]", transactionCount, context, this.sessionId), new Exception("Uncommitted repository changes")); + LOGGER.warn(() -> String.format("Uncommitted changes (%d) after %s, session [%d]", + transactionCount, + context, + this.sessionId), + new Exception("Uncommitted repository changes")); logStatements(); } } @@ -760,4 +1013,55 @@ public static String ed25519PublicKeyToAddress(byte[] publicKey) { return Crypto.toAddress(publicKey); } -} \ No newline at end of file + /*package*/ static boolean isDeadlockException(SQLException e) { + return DEADLOCK_ERROR_CODE.equals(e.getErrorCode()); + } + + private int otherTransactionsCount() throws DataException { + // We can only perform a CHECKPOINT if no other HSQLDB session is mid-transaction, + // otherwise the CHECKPOINT blocks for COMMITs and other threads can't open HSQLDB sessions + // due to HSQLDB blocking until CHECKPOINT finishes - i.e. deadlock + String sql = "SELECT COUNT(*) " + + "FROM Information_schema.system_sessions " + + "WHERE transaction = TRUE AND session_id != ?"; + try { + PreparedStatement pstmt = this.cachePreparedStatement(sql); + pstmt.setLong(1, this.sessionId); + + if (!pstmt.execute()) + throw new DataException("Unable to check repository session status"); + + try (ResultSet resultSet = pstmt.getResultSet()) { + if (resultSet == null || !resultSet.next()) + // Failed to even find HSQLDB session info! + throw new DataException("No results when checking repository session status"); + + int transactionCount = resultSet.getInt(1); + + return transactionCount; + } + } catch (SQLException e) { + throw new DataException("Unable to check repository session status", e); + } + } + + private void blockUntilNoOtherTransactions(Long timeout) throws DataException, TimeoutException { + try { + long startTime = System.currentTimeMillis(); + while (this.otherTransactionsCount() > 0) { + // Wait and try again + LOGGER.debug("Waiting for repository..."); + Thread.sleep(1000L); + + if (timeout != null) { + if (System.currentTimeMillis() - startTime >= timeout) { + throw new TimeoutException("Timed out waiting for repository to become available"); + } + } + } + } catch (InterruptedException e) { + throw new DataException("Interrupted before repository became available"); + } + } + +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java index 8561b6980..64f6be8c7 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java @@ -14,17 +14,18 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryFactory; +import org.qortal.settings.Settings; public class HSQLDBRepositoryFactory implements RepositoryFactory { private static final Logger LOGGER = LogManager.getLogger(HSQLDBRepositoryFactory.class); - private static final int POOL_SIZE = 100; /** Log getConnection() calls that take longer than this. (ms) */ private static final long SLOW_CONNECTION_THRESHOLD = 1000L; private String connectionUrl; private HSQLDBPool connectionPool; + private final boolean wasPristine; /** * Constructs new RepositoryFactory using passed connectionUrl. @@ -53,10 +54,10 @@ public HSQLDBRepositoryFactory(String connectionUrl) throws DataException { throw new DataException("Unable to read repository: " + e.getMessage(), e); // Attempt recovery? - HSQLDBRepository.attemptRecovery(connectionUrl); + HSQLDBRepository.attemptRecovery(connectionUrl, "backup"); } - this.connectionPool = new HSQLDBPool(POOL_SIZE); + this.connectionPool = new HSQLDBPool(Settings.getInstance().getRepositoryConnectionPoolSize()); this.connectionPool.setUrl(this.connectionUrl); Properties properties = new Properties(); @@ -65,12 +66,17 @@ public HSQLDBRepositoryFactory(String connectionUrl) throws DataException { // Perform DB updates? try (final Connection connection = this.connectionPool.getConnection()) { - HSQLDBDatabaseUpdates.updateDatabase(connection); + this.wasPristine = HSQLDBDatabaseUpdates.updateDatabase(connection); } catch (SQLException e) { throw new DataException("Repository initialization error", e); } } + @Override + public boolean wasPristineAtOpen() { + return this.wasPristine; + } + @Override public RepositoryFactory reopen() throws DataException { return new HSQLDBRepositoryFactory(this.connectionUrl); @@ -88,7 +94,11 @@ public Repository getRepository() throws DataException { @Override public Repository tryRepository() throws DataException { try { - return new HSQLDBRepository(this.tryConnection()); + Connection connection = this.tryConnection(); + if (connection == null) + return null; + + return new HSQLDBRepository(connection); } catch (SQLException e) { throw new DataException("Repository instantiation error", e); } @@ -138,4 +148,9 @@ public void close() throws DataException { } } + @Override + public boolean isDeadlockException(SQLException e) { + return HSQLDBRepository.isDeadlockException(e); + } + } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBSaver.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBSaver.java index 8384c22f4..acf24c547 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBSaver.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBSaver.java @@ -60,12 +60,16 @@ public HSQLDBSaver bind(String column, Object value) { */ public boolean execute(HSQLDBRepository repository) throws SQLException { String sql = this.formatInsertWithPlaceholders(); - try (PreparedStatement preparedStatement = repository.prepareStatement(sql)) { - this.bindValues(preparedStatement); - return preparedStatement.execute(); - } catch (SQLException e) { - throw repository.examineException(e); + synchronized (HSQLDBRepository.CHECKPOINT_LOCK) { + try { + PreparedStatement preparedStatement = repository.prepareStatement(sql); + this.bindValues(preparedStatement); + + return preparedStatement.execute(); + } catch (SQLException e) { + throw repository.examineException(e); + } } } diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java index 804b2b106..c7f4c9588 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java @@ -4,6 +4,7 @@ import java.sql.SQLException; import java.util.List; +import org.qortal.arbitrary.misc.Service; import org.qortal.data.PaymentData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.BaseTransactionData; @@ -20,21 +21,31 @@ public HSQLDBArbitraryTransactionRepository(HSQLDBRepository repository) { } TransactionData fromBase(BaseTransactionData baseTransactionData) throws DataException { - String sql = "SELECT version, service, is_data_raw, data from ArbitraryTransactions WHERE signature = ?"; + String sql = "SELECT version, nonce, service, size, is_data_raw, data, metadata_hash, " + + "name, identifier, update_method, secret, compression from ArbitraryTransactions " + + "WHERE signature = ?"; try (ResultSet resultSet = this.repository.checkedExecute(sql, baseTransactionData.getSignature())) { if (resultSet == null) return null; int version = resultSet.getInt(1); - int service = resultSet.getInt(2); - boolean isDataRaw = resultSet.getBoolean(3); // NOT NULL, so no null to false + int nonce = resultSet.getInt(2); + Service service = Service.valueOf(resultSet.getInt(3)); + int size = resultSet.getInt(4); + boolean isDataRaw = resultSet.getBoolean(5); // NOT NULL, so no null to false DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; - byte[] data = resultSet.getBytes(4); + byte[] data = resultSet.getBytes(6); + byte[] metadataHash = resultSet.getBytes(7); + String name = resultSet.getString(8); + String identifier = resultSet.getString(9); + ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.valueOf(resultSet.getInt(10)); + byte[] secret = resultSet.getBytes(11); + ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.valueOf(resultSet.getInt(12)); List payments = this.getPaymentsFromSignature(baseTransactionData.getSignature()); - - return new ArbitraryTransactionData(baseTransactionData, version, service, data, dataType, payments); + return new ArbitraryTransactionData(baseTransactionData, version, service, nonce, size, name, + identifier, method, secret, compression, data, dataType, metadataHash, payments); } catch (SQLException e) { throw new DataException("Unable to fetch arbitrary transaction from repository", e); } @@ -48,11 +59,19 @@ public void save(TransactionData transactionData) throws DataException { if (arbitraryTransactionData.getVersion() >= 4) this.repository.getArbitraryRepository().save(arbitraryTransactionData); + // method and compression use NOT NULL DEFAULT 0, so fall back to these values if null + Integer method = arbitraryTransactionData.getMethod() != null ? arbitraryTransactionData.getMethod().value : 0; + Integer compression = arbitraryTransactionData.getCompression() != null ? arbitraryTransactionData.getCompression().value : 0; + HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryTransactions"); saveHelper.bind("signature", arbitraryTransactionData.getSignature()).bind("sender", arbitraryTransactionData.getSenderPublicKey()) - .bind("version", arbitraryTransactionData.getVersion()).bind("service", arbitraryTransactionData.getService()) - .bind("is_data_raw", arbitraryTransactionData.getDataType() == DataType.RAW_DATA).bind("data", arbitraryTransactionData.getData()); + .bind("version", arbitraryTransactionData.getVersion()).bind("service", arbitraryTransactionData.getService().value) + .bind("nonce", arbitraryTransactionData.getNonce()).bind("size", arbitraryTransactionData.getSize()) + .bind("is_data_raw", arbitraryTransactionData.getDataType() == DataType.RAW_DATA).bind("data", arbitraryTransactionData.getData()) + .bind("metadata_hash", arbitraryTransactionData.getMetadataHash()).bind("name", arbitraryTransactionData.getName()) + .bind("identifier", arbitraryTransactionData.getIdentifier()).bind("update_method", method) + .bind("secret", arbitraryTransactionData.getSecret()).bind("compression", compression); try { saveHelper.execute(this.repository); diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBPresenceTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBPresenceTransactionRepository.java new file mode 100644 index 000000000..309ffcad5 --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBPresenceTransactionRepository.java @@ -0,0 +1,57 @@ +package org.qortal.repository.hsqldb.transaction; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.hsqldb.HSQLDBRepository; +import org.qortal.repository.hsqldb.HSQLDBSaver; +import org.qortal.transaction.PresenceTransaction.PresenceType; + +public class HSQLDBPresenceTransactionRepository extends HSQLDBTransactionRepository { + + public HSQLDBPresenceTransactionRepository(HSQLDBRepository repository) { + this.repository = repository; + } + + TransactionData fromBase(BaseTransactionData baseTransactionData) throws DataException { + String sql = "SELECT nonce, presence_type, timestamp_signature FROM PresenceTransactions WHERE signature = ?"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, baseTransactionData.getSignature())) { + if (resultSet == null) + return null; + + int nonce = resultSet.getInt(1); + int presenceTypeValue = resultSet.getInt(2); + PresenceType presenceType = PresenceType.valueOf(presenceTypeValue); + + byte[] timestampSignature = resultSet.getBytes(3); + + return new PresenceTransactionData(baseTransactionData, nonce, presenceType, timestampSignature); + } catch (SQLException e) { + throw new DataException("Unable to fetch presence transaction from repository", e); + } + } + + @Override + public void save(TransactionData transactionData) throws DataException { + PresenceTransactionData presenceTransactionData = (PresenceTransactionData) transactionData; + + HSQLDBSaver saveHelper = new HSQLDBSaver("PresenceTransactions"); + + saveHelper.bind("signature", presenceTransactionData.getSignature()) + .bind("nonce", presenceTransactionData.getNonce()) + .bind("presence_type", presenceTransactionData.getPresenceType().value) + .bind("timestamp_signature", presenceTransactionData.getTimestampSignature()); + + try { + saveHelper.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save chat transaction into repository", e); + } + } + +} diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index 0ab6ed94e..f228944ee 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -9,12 +9,14 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.EnumMap; +import java.util.EnumSet; import java.util.List; import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; +import org.qortal.arbitrary.misc.Service; import org.qortal.data.PaymentData; import org.qortal.data.group.GroupApprovalData; import org.qortal.data.transaction.BaseTransactionData; @@ -28,6 +30,7 @@ import org.qortal.transaction.Transaction.ApprovalStatus; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.utils.Base58; +import org.qortal.utils.Unicode; public class HSQLDBTransactionRepository implements TransactionRepository { @@ -385,8 +388,8 @@ public Map getTransactionSummary(int startHeight, int @Override public List getSignaturesMatchingCriteria(Integer startBlock, Integer blockLimit, Integer txGroupId, - List txTypes, Integer service, String address, - ConfirmationStatus confirmationStatus, Integer limit, Integer offset, Boolean reverse) throws DataException { + List txTypes, Service service, String name, String address, + ConfirmationStatus confirmationStatus, Integer limit, Integer offset, Boolean reverse) throws DataException { List signatures = new ArrayList<>(); boolean hasAddress = address != null && !address.isEmpty(); @@ -411,8 +414,8 @@ public List getSignaturesMatchingCriteria(Integer startBlock, Integer bl signatureColumn = "TransactionParticipants.signature"; } - if (service != null) { - // This is for ARBITRARY transactions + if (service != null || name != null) { + // These are for ARBITRARY transactions tables.append(" LEFT OUTER JOIN ArbitraryTransactions ON ArbitraryTransactions.signature = Transactions.signature"); } @@ -465,7 +468,12 @@ public List getSignaturesMatchingCriteria(Integer startBlock, Integer bl if (service != null) { whereClauses.add("ArbitraryTransactions.service = ?"); - bindParams.add(service); + bindParams.add(service.value); + } + + if (name != null) { + whereClauses.add("lower(ArbitraryTransactions.name) = ?"); + bindParams.add(name.toLowerCase()); } if (hasAddress) { @@ -585,6 +593,107 @@ public List getSignaturesMatchingCriteria(TransactionType txType, byte[] } } + @Override + public List getSignaturesMatchingCriteria(TransactionType txType, byte[] publicKey, + Integer minBlockHeight, Integer maxBlockHeight) throws DataException { + List signatures = new ArrayList<>(); + + StringBuilder sql = new StringBuilder(1024); + sql.append("SELECT signature FROM Transactions "); + + List whereClauses = new ArrayList<>(); + List bindParams = new ArrayList<>(); + + if (txType != null) { + whereClauses.add("type = ?"); + bindParams.add(txType.value); + } + + if (publicKey != null) { + whereClauses.add("creator = ?"); + bindParams.add(publicKey); + } + + if (minBlockHeight != null) { + whereClauses.add("Transactions.block_height >= ?"); + bindParams.add(minBlockHeight); + } + + if (maxBlockHeight != null) { + whereClauses.add("Transactions.block_height <= ?"); + bindParams.add(maxBlockHeight); + } + + if (!whereClauses.isEmpty()) { + sql.append(" WHERE "); + + final int whereClausesSize = whereClauses.size(); + for (int wci = 0; wci < whereClausesSize; ++wci) { + if (wci != 0) + sql.append(" AND "); + + sql.append(whereClauses.get(wci)); + } + } + + sql.append(" ORDER BY Transactions.created_when"); + + LOGGER.trace(() -> String.format("Transaction search SQL: %s", sql)); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return signatures; + + do { + byte[] signature = resultSet.getBytes(1); + + signatures.add(signature); + } while (resultSet.next()); + + return signatures; + } catch (SQLException e) { + throw new DataException("Unable to fetch matching transaction signatures from repository", e); + } + } + + + public List getSignaturesMatchingCustomCriteria(TransactionType txType, List whereClauses, + List bindParams) throws DataException { + List signatures = new ArrayList<>(); + + StringBuilder sql = new StringBuilder(1024); + sql.append(String.format("SELECT signature FROM %sTransactions", txType.className)); + + if (!whereClauses.isEmpty()) { + sql.append(" WHERE "); + + final int whereClausesSize = whereClauses.size(); + for (int wci = 0; wci < whereClausesSize; ++wci) { + if (wci != 0) + sql.append(" AND "); + + sql.append(whereClauses.get(wci)); + } + } + + LOGGER.trace(() -> String.format("Transaction search SQL: %s", sql)); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return signatures; + + do { + byte[] signature = resultSet.getBytes(1); + + signatures.add(signature); + } while (resultSet.next()); + + return signatures; + } catch (SQLException e) { + throw new DataException("Unable to fetch matching transaction signatures from repository", e); + } + } + @Override public byte[] getLatestAutoUpdateTransaction(TransactionType txType, int txGroupId, Integer service) throws DataException { StringBuilder sql = new StringBuilder(1024); @@ -630,6 +739,88 @@ public byte[] getLatestAutoUpdateTransaction(TransactionType txType, int txGroup } } + @Override + public List getTransactionsInvolvingName(String name, ConfirmationStatus confirmationStatus) throws DataException { + TransactionType[] transactionTypes = new TransactionType[] { + REGISTER_NAME, UPDATE_NAME, BUY_NAME, SELL_NAME + }; // TODO: CancelSellNameTransaction? + + String reducedName = Unicode.sanitize(name); + + StringBuilder sql = new StringBuilder(1024); + List bindParams = new ArrayList<>(); + sql.append("SELECT Transactions.signature FROM Transactions"); + + for (int ti = 0; ti < transactionTypes.length; ++ti) { + sql.append(" LEFT OUTER JOIN "); + sql.append(transactionTypes[ti].className); + sql.append("Transactions USING (signature)"); + } + + sql.append(" WHERE Transactions.type IN ("); + for (int ti = 0; ti < transactionTypes.length; ++ti) { + if (ti != 0) + sql.append(", "); + + sql.append(transactionTypes[ti].value); + } + sql.append(")"); + + // Confirmation status + switch (confirmationStatus) { + case BOTH: + break; + + case CONFIRMED: + sql.append(" AND Transactions.block_height IS NOT NULL"); + break; + + case UNCONFIRMED: + sql.append(" AND Transactions.block_height IS NULL"); + break; + } + + sql.append(" AND (RegisterNameTransactions.name = ?"); + bindParams.add(name); + sql.append(" OR RegisterNameTransactions.reduced_name = ?"); + bindParams.add(reducedName); + sql.append(" OR UpdateNameTransactions.name = ?"); + bindParams.add(name); + sql.append(" OR (UpdateNameTransactions.reduced_new_name != '' AND UpdateNameTransactions.reduced_new_name = ?)"); + bindParams.add(reducedName); + sql.append(" OR UpdateNameTransactions.new_name = ?"); + bindParams.add(name); + sql.append(" OR SellNameTransactions.name = ?"); + bindParams.add(name); + sql.append(" OR BuyNameTransactions.name = ?"); + bindParams.add(name); + + sql.append(") GROUP BY Transactions.signature, Transactions.created_when ORDER BY Transactions.created_when"); + + List transactions = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return transactions; + + do { + byte[] signature = resultSet.getBytes(1); + + TransactionData transactionData = this.fromSignature(signature); + + if (transactionData == null) + // Something inconsistent with the repository + throw new DataException("Unable to fetch name-related transaction from repository?"); + + transactions.add(transactionData); + } while (resultSet.next()); + + return transactions; + } catch (SQLException | DataException e) { + throw new DataException("Unable to fetch name-related transactions from repository", e); + } + } + @Override public List getAssetTransactions(long assetId, ConfirmationStatus confirmationStatus, Integer limit, Integer offset, Boolean reverse) throws DataException { @@ -1061,6 +1252,108 @@ public List getUnconfirmedTransactions(Integer limit, Integer o } } + @Override + public List getUnconfirmedTransactions(TransactionType txType, byte[] creatorPublicKey) throws DataException { + if (txType == null && creatorPublicKey == null) + throw new IllegalArgumentException("At least one of txType or creatorPublicKey must be non-null"); + + StringBuilder sql = new StringBuilder(1024); + sql.append("SELECT signature FROM UnconfirmedTransactions "); + sql.append("JOIN Transactions USING (signature) "); + sql.append("WHERE "); + + List whereClauses = new ArrayList<>(); + List bindParams = new ArrayList<>(); + + if (txType != null) { + whereClauses.add("type = ?"); + bindParams.add(Integer.valueOf(txType.value)); + } + + if (creatorPublicKey != null) { + whereClauses.add("creator = ?"); + bindParams.add(creatorPublicKey); + } + + final int whereClausesSize = whereClauses.size(); + for (int wci = 0; wci < whereClausesSize; ++wci) { + if (wci != 0) + sql.append(" AND "); + + sql.append(whereClauses.get(wci)); + } + + sql.append("ORDER BY created_when, signature"); + + List transactions = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return transactions; + + do { + byte[] signature = resultSet.getBytes(1); + + TransactionData transactionData = this.fromSignature(signature); + + if (transactionData == null) + // Something inconsistent with the repository + throw new DataException(String.format("Unable to fetch unconfirmed transaction %s from repository?", Base58.encode(signature))); + + transactions.add(transactionData); + } while (resultSet.next()); + + return transactions; + } catch (SQLException | DataException e) { + throw new DataException("Unable to fetch unconfirmed transactions from repository", e); + } + } + + @Override + public List getUnconfirmedTransactions(EnumSet excludedTxTypes) throws DataException { + StringBuilder sql = new StringBuilder(1024); + sql.append("SELECT signature FROM UnconfirmedTransactions "); + sql.append("JOIN Transactions USING (signature) "); + sql.append("WHERE type NOT IN ("); + + boolean firstTxType = true; + for (TransactionType txType : excludedTxTypes) { + if (firstTxType) + firstTxType = false; + else + sql.append(", "); + + sql.append(txType.value); + } + + sql.append(")"); + sql.append("ORDER BY created_when, signature"); + + List transactions = new ArrayList<>(); + + // Find transactions with no corresponding row in BlockTransactions + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) { + if (resultSet == null) + return transactions; + + do { + byte[] signature = resultSet.getBytes(1); + + TransactionData transactionData = this.fromSignature(signature); + + if (transactionData == null) + // Something inconsistent with the repository + throw new DataException(String.format("Unable to fetch unconfirmed transaction %s from repository?", Base58.encode(signature))); + + transactions.add(transactionData); + } while (resultSet.next()); + + return transactions; + } catch (SQLException | DataException e) { + throw new DataException("Unable to fetch unconfirmed transactions from repository", e); + } + } + @Override public void confirmTransaction(byte[] signature) throws DataException { try { diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 84483b9ee..779c29f53 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -5,6 +5,8 @@ import java.io.FileReader; import java.io.IOException; import java.io.Reader; +import java.nio.file.Paths; +import java.util.*; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; @@ -20,7 +22,11 @@ import org.eclipse.persistence.jaxb.JAXBContextFactory; import org.eclipse.persistence.jaxb.UnmarshallerProperties; import org.qortal.block.BlockChain; -import org.qortal.crosschain.BTC.BitcoinNet; +import org.qortal.controller.arbitrary.ArbitraryDataStorageManager.*; +import org.qortal.crosschain.Bitcoin.BitcoinNet; +import org.qortal.crosschain.Litecoin.LitecoinNet; +import org.qortal.crosschain.Dogecoin.DogecoinNet; +import org.qortal.utils.EnumUtils; // All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) @@ -32,6 +38,12 @@ public class Settings { private static final int MAINNET_API_PORT = 12391; private static final int TESTNET_API_PORT = 62391; + private static final int MAINNET_DOMAIN_MAP_PORT = 80; + private static final int TESTNET_DOMAIN_MAP_PORT = 8080; + + private static final int MAINNET_GATEWAY_PORT = 80; + private static final int TESTNET_GATEWAY_PORT = 8080; + private static final Logger LOGGER = LogManager.getLogger(Settings.class); private static final String SETTINGS_FILENAME = "settings.json"; @@ -41,13 +53,16 @@ public class Settings { // Settings, and other config files private String userPath; + // General + private String localeLang = Locale.getDefault().getLanguage(); + // Common to all networking (API/P2P) private String bindAddress = "::"; // Use IPv6 wildcard to listen on all local addresses // UI servers private int uiPort = 12388; private String[] uiLocalServers = new String[] { - "localhost", "127.0.0.1", "172.24.1.1", "qor.tal" + "localhost", "127.0.0.1" }; private String[] uiRemoteServers = new String[] { "node1.qortal.org", "node2.qortal.org", "node3.qortal.org", "node4.qortal.org", "node5.qortal.org", @@ -57,9 +72,18 @@ public class Settings { // API-related private boolean apiEnabled = true; private Integer apiPort; + private boolean apiWhitelistEnabled = true; private String[] apiWhitelist = new String[] { "::1", "127.0.0.1" }; + + /** Legacy API key (deprecated Nov 2021). Use /admin/apikey/generate API endpoint instead */ + private String apiKey = null; + /** Storage location for API key generated by API (Nov 2021 onwards) */ + private String apiKeyPath = ""; + /** Whether to allow automatic authentication from localhost (loopback) addresses */ + private boolean localAuthBypassEnabled = false; + private Boolean apiRestricted; private boolean apiLoggingEnabled = false; private boolean apiDocumentationEnabled = false; @@ -67,6 +91,17 @@ public class Settings { private String sslKeystorePathname = null; private String sslKeystorePassword = null; + // Domain mapping + private Integer domainMapPort; + private boolean domainMapEnabled = false; + private boolean domainMapLoggingEnabled = false; + private List domainMap = null; + + // Gateway + private Integer gatewayPort; + private boolean gatewayEnabled = false; + private boolean gatewayLoggingEnabled = false; + // Specific to this node private boolean wipeUnconfirmedOnStart = false; /** Maximum number of unconfirmed transactions allowed per account */ @@ -79,11 +114,75 @@ public class Settings { private long repositoryBackupInterval = 0; // ms /** Whether to show a notification when we backup repository. */ private boolean showBackupNotification = false; + /** Minimum time between repository maintenance attempts (ms) */ + private long repositoryMaintenanceMinInterval = 3 * 24 * 60 * 60 * 1000L; // 3 days (ms) default + /** Maximum time between repository maintenance attempts (ms) (0 if disabled). */ + private long repositoryMaintenanceMaxInterval = 14 * 24 * 60 * 60 * 1000L; // 14 days (ms) default + /** Whether to show a notification when we run scheduled maintenance. */ + private boolean showMaintenanceNotification = false; + /** How long between repository checkpoints (ms). */ + private long repositoryCheckpointInterval = 60 * 60 * 1000L; // 1 hour (ms) default + /** Whether to show a notification when we perform repository 'checkpoint'. */ + private boolean showCheckpointNotification = false; + /* How many blocks to cache locally. Defaulted to 10, which covers a typical Synchronizer request + a few spare */ + private int blockCacheSize = 10; + + /** How long to keep old, full, AT state data (ms). */ + private long atStatesMaxLifetime = 5 * 24 * 60 * 60 * 1000L; // milliseconds + /** How often to attempt AT state trimming (ms). */ + private long atStatesTrimInterval = 5678L; // milliseconds + /** Block height range to scan for trimmable AT states.
+ * This has a significant effect on execution time. */ + private int atStatesTrimBatchSize = 100; // blocks + /** Max number of AT states to trim in one go. */ + private int atStatesTrimLimit = 4000; // records + + /** How often to attempt online accounts signatures trimming (ms). */ + private long onlineSignaturesTrimInterval = 9876L; // milliseconds + /** Block height range to scan for trimmable online accounts signatures.
+ * This has a significant effect on execution time. */ + private int onlineSignaturesTrimBatchSize = 100; // blocks + + + /** Whether we should prune old data to reduce database size + * This prevents the node from being able to serve older blocks */ + private boolean topOnly = false; + /** The amount of recent blocks we should keep when pruning */ + private int pruneBlockLimit = 1450; + + /** How often to attempt AT state pruning (ms). */ + private long atStatesPruneInterval = 3219L; // milliseconds + /** Block height range to scan for prunable AT states.
+ * This has a significant effect on execution time. */ + private int atStatesPruneBatchSize = 25; // blocks + + /** How often to attempt block pruning (ms). */ + private long blockPruneInterval = 3219L; // milliseconds + /** Block height range to scan for prunable blocks.
+ * This has a significant effect on execution time. */ + private int blockPruneBatchSize = 10000; // blocks + + + /** Whether we should archive old data to reduce the database size */ + private boolean archiveEnabled = true; + /** How often to attempt archiving (ms). */ + private long archiveInterval = 7171L; // milliseconds + + + /** Whether to automatically bootstrap instead of syncing from genesis */ + private boolean bootstrap = true; + + + /** Registered names integrity check */ + private boolean namesIntegrityCheckEnabled = false; + // Peer-to-peer related private boolean isTestNet = false; /** Port number for inbound peer-to-peer connections. */ private Integer listenPort; + /** Whether to attempt to open the listen port via UPnP */ + private boolean uPnPEnabled = true; /** Minimum number of peers to allow block minting / synchronization. */ private int minBlockchainPeers = 5; /** Target number of outbound connections to peers we should make. */ @@ -94,25 +193,71 @@ public class Settings { private int maxNetworkThreadPoolSize = 20; /** Maximum number of threads for network proof-of-work compute, used during handshaking. */ private int networkPoWComputePoolSize = 2; + /** Maximum number of retry attempts if a peer fails to respond with the requested data */ + private int maxRetries = 2; + + /** Minimum peer version number required in order to sync with them */ + private String minPeerVersion = "3.1.0"; + /** Whether to allow connections with peers below minPeerVersion + * If true, we won't sync with them but they can still sync with us, and will show in the peers list + * If false, sync will be blocked both ways, and they will not appear in the peers list */ + private boolean allowConnectionsWithOlderPeerVersions = true; + + /** Minimum time (in seconds) that we should attempt to remain connected to a peer for */ + private int minPeerConnectionTime = 5 * 60; // seconds + /** Maximum time (in seconds) that we should attempt to remain connected to a peer for */ + private int maxPeerConnectionTime = 60 * 60; // seconds + + /** Whether to sync multiple blocks at once in normal operation */ + private boolean fastSyncEnabled = true; + /** Whether to sync multiple blocks at once when the peer has a different chain */ + private boolean fastSyncEnabledWhenResolvingFork = true; + /** Maximum number of blocks to request at once */ + private int maxBlocksPerRequest = 100; + /** Maximum number of blocks this node will serve in a single response */ + private int maxBlocksPerResponse = 200; // Which blockchains this node is running private String blockchainConfig = null; // use default from resources private BitcoinNet bitcoinNet = BitcoinNet.MAIN; + private LitecoinNet litecoinNet = LitecoinNet.MAIN; + private DogecoinNet dogecoinNet = DogecoinNet.MAIN; + // Also crosschain-related: + /** Whether to show SysTray pop-up notifications when trade-bot entries change state */ + private boolean tradebotSystrayEnabled = false; // Repository related /** Queries that take longer than this are logged. (milliseconds) */ private Long slowQueryThreshold = null; /** Repository storage path. */ private String repositoryPath = "db"; + /** Repository connection pool size. Needs to be a bit bigger than maxNetworkThreadPoolSize */ + private int repositoryConnectionPoolSize = 100; + private List fixedNetwork; + + // Export/import + private String exportPath = "qortal-backup"; + + // Bootstrap + private String bootstrapFilenamePrefix = ""; + + // Bootstrap sources + private String[] bootstrapHosts = new String[] { + "http://bootstrap.qortal.org", + "http://bootstrap2.qortal.org", + "http://81.169.136.59", + "http://62.171.190.193" + }; // Auto-update sources private String[] autoUpdateRepos = new String[] { - "https://github.com/QORT/qortal/raw/%s/qortal.update", - "https://raw.githubusercontent.com@151.101.16.133/QORT/qortal/%s/qortal.update", "https://github.com/Qortal/qortal/raw/%s/qortal.update", "https://raw.githubusercontent.com@151.101.16.133/Qortal/qortal/%s/qortal.update" }; + // Lists + private String listsPath = "lists"; + /** Array of NTP server hostnames. */ private String[] ntpServers = new String[] { "pool.ntp.org", @@ -129,6 +274,72 @@ public class Settings { /** Additional offset added to values returned by NTP.getTime() */ private Long testNtpOffset = null; + + // Data storage (QDN) + + /** Data storage enabled/disabled*/ + private boolean qdnEnabled = true; + /** Data storage path. */ + private String dataPath = "data"; + /** Data storage path (for temporary data). Defaults to {dataPath}/_temp */ + private String tempDataPath = null; + + /** Storage policy to indicate which data should be hosted */ + private String storagePolicy = "FOLLOWED_OR_VIEWED"; + + /** Whether to allow data outside of the storage policy to be relayed between other peers */ + private boolean relayModeEnabled = true; + + /** Whether to remember which data was originally uploaded using this node. + * This prevents auto deletion of own files when storage limits are reached. */ + private boolean originalCopyIndicatorFileEnabled = true; + + /** Whether to make connections directly with peers that have the required data */ + private boolean directDataRetrievalEnabled = true; + + /** Expiry time (ms) for (unencrypted) built/cached data */ + private Long builtDataExpiryInterval = 30 * 24 * 60 * 60 * 1000L; // 30 days + + /** Whether to validate every layer when building arbitrary data, or just the final layer */ + private boolean validateAllDataLayers = false; + + /** Whether to allow public (decryptable) data to be stored */ + private boolean publicDataEnabled = true; + /** Whether to allow private (non-decryptable) data to be stored */ + private boolean privateDataEnabled = false; + + /** Maximum total size of hosted data, in bytes. Unlimited if null */ + private Long maxStorageCapacity = null; + + /** Whether to serve QDN data without authentication */ + private boolean qdnAuthBypassEnabled = false; + + // Domain mapping + public static class DomainMap { + private String domain; + private String name; + + private DomainMap() { // makes JAXB happy; will never be invoked + } + + public String getDomain() { + return domain; + } + + public void setDomain(String domain) { + this.domain = domain; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + // Constructors private Settings() { @@ -250,6 +461,16 @@ private void validate() { // Validation goes here if (this.minBlockchainPeers < 1) throwValidationError("minBlockchainPeers must be at least 1"); + + if (this.apiKey != null && this.apiKey.trim().length() < 8) + throwValidationError("apiKey must be at least 8 characters"); + + try { + StoragePolicy.valueOf(this.storagePolicy); + } catch (IllegalArgumentException ex) { + String possibleValues = EnumUtils.getNames(StoragePolicy.class, ", "); + throwValidationError(String.format("storagePolicy must be one of: %s", possibleValues)); + } } // Getters / setters @@ -258,6 +479,10 @@ public String getUserPath() { return this.userPath; } + public String getLocaleLang() { + return this.localeLang; + } + public int getUiServerPort() { return this.uiPort; } @@ -282,6 +507,10 @@ public int getApiPort() { } public String[] getApiWhitelist() { + if (!this.apiWhitelistEnabled) { + // Allow all connections if the whitelist is disabled + return new String[] {"0.0.0.0/0", "::/0"}; + } return this.apiWhitelist; } @@ -294,6 +523,18 @@ public boolean isApiRestricted() { return !BlockChain.getInstance().isTestChain(); } + public String getApiKey() { + return this.apiKey; + } + + public String getApiKeyPath() { + return this.apiKeyPath; + } + + public boolean isLocalAuthBypassEnabled() { + return this.localAuthBypassEnabled; + } + public boolean isApiLoggingEnabled() { return this.apiLoggingEnabled; } @@ -310,6 +551,51 @@ public String getSslKeystorePassword() { return this.sslKeystorePassword; } + public int getDomainMapPort() { + if (this.domainMapPort != null) + return this.domainMapPort; + + return this.isTestNet ? TESTNET_DOMAIN_MAP_PORT : MAINNET_DOMAIN_MAP_PORT; + } + + public boolean isDomainMapEnabled() { + return this.domainMapEnabled; + } + + public boolean isDomainMapLoggingEnabled() { + return this.domainMapLoggingEnabled; + } + + public Map getSimpleDomainMap() { + HashMap map = new HashMap<>(); + for (DomainMap dMap : this.domainMap) { + map.put(dMap.getDomain(), dMap.getName()); + + // If the domain doesn't include a subdomain then add a www. alternative + if (dMap.getDomain().chars().filter(c -> c == '.').count() == 1) { + map.put("www.".concat(dMap.getDomain()), dMap.getName()); + } + } + return map; + } + + + public int getGatewayPort() { + if (this.gatewayPort != null) + return this.gatewayPort; + + return this.isTestNet ? TESTNET_GATEWAY_PORT : MAINNET_GATEWAY_PORT; + } + + public boolean isGatewayEnabled() { + return this.gatewayEnabled; + } + + public boolean isGatewayLoggingEnabled() { + return this.gatewayLoggingEnabled; + } + + public boolean getWipeUnconfirmedOnStart() { return this.wipeUnconfirmedOnStart; } @@ -322,6 +608,10 @@ public int getMaxTransactionTimestampFuture() { return this.maxTransactionTimestampFuture; } + public int getBlockCacheSize() { + return this.blockCacheSize; + } + public boolean isTestNet() { return this.isTestNet; } @@ -341,6 +631,10 @@ public String getBindAddress() { return this.bindAddress; } + public boolean isUPnPEnabled() { + return this.uPnPEnabled; + } + public int getMinBlockchainPeers() { return this.minBlockchainPeers; } @@ -361,6 +655,16 @@ public int getNetworkPoWComputePoolSize() { return this.networkPoWComputePoolSize; } + public int getMaxRetries() { return this.maxRetries; } + + public String getMinPeerVersion() { return this.minPeerVersion; } + + public boolean getAllowConnectionsWithOlderPeerVersions() { return this.allowConnectionsWithOlderPeerVersions; } + + public int getMinPeerConnectionTime() { return this.minPeerConnectionTime; } + + public int getMaxPeerConnectionTime() { return this.maxPeerConnectionTime; } + public String getBlockchainConfig() { return this.blockchainConfig; } @@ -369,6 +673,18 @@ public BitcoinNet getBitcoinNet() { return this.bitcoinNet; } + public LitecoinNet getLitecoinNet() { + return this.litecoinNet; + } + + public DogecoinNet getDogecoinNet() { + return this.dogecoinNet; + } + + public boolean isTradebotSystrayEnabled() { + return this.tradebotSystrayEnabled; + } + public Long getSlowQueryThreshold() { return this.slowQueryThreshold; } @@ -377,6 +693,30 @@ public String getRepositoryPath() { return this.repositoryPath; } + public int getRepositoryConnectionPoolSize() { + return this.repositoryConnectionPoolSize; + } + + public String getExportPath() { + return this.exportPath; + } + + public String getBootstrapFilenamePrefix() { + return this.bootstrapFilenamePrefix; + } + + public boolean isFastSyncEnabled() { + return this.fastSyncEnabled; + } + + public boolean isFastSyncEnabledWhenResolvingFork() { + return this.fastSyncEnabledWhenResolvingFork; + } + + public int getMaxBlocksPerRequest() { return this.maxBlocksPerRequest; } + + public int getMaxBlocksPerResponse() { return this.maxBlocksPerResponse; } + public boolean isAutoUpdateEnabled() { return this.autoUpdateEnabled; } @@ -385,6 +725,14 @@ public String[] getAutoUpdateRepos() { return this.autoUpdateRepos; } + public String[] getBootstrapHosts() { + return this.bootstrapHosts; + } + + public String getListsPath() { + return this.listsPath; + } + public String[] getNtpServers() { return this.ntpServers; } @@ -401,4 +749,153 @@ public boolean getShowBackupNotification() { return this.showBackupNotification; } + public long getRepositoryMaintenanceMinInterval() { + return this.repositoryMaintenanceMinInterval; + } + + public long getRepositoryMaintenanceMaxInterval() { + return this.repositoryMaintenanceMaxInterval; + } + + public boolean getShowMaintenanceNotification() { + return this.showMaintenanceNotification; + } + + public long getRepositoryCheckpointInterval() { + return this.repositoryCheckpointInterval; + } + + public boolean getShowCheckpointNotification() { + return this.showCheckpointNotification; + } + + public List getFixedNetwork() { + return fixedNetwork; + } + + public long getAtStatesMaxLifetime() { + return this.atStatesMaxLifetime; + } + + public long getAtStatesTrimInterval() { + return this.atStatesTrimInterval; + } + + public int getAtStatesTrimBatchSize() { + return this.atStatesTrimBatchSize; + } + + public int getAtStatesTrimLimit() { + return this.atStatesTrimLimit; + } + + public long getOnlineSignaturesTrimInterval() { + return this.onlineSignaturesTrimInterval; + } + + public int getOnlineSignaturesTrimBatchSize() { + return this.onlineSignaturesTrimBatchSize; + } + + public boolean isTopOnly() { + return this.topOnly; + } + + public int getPruneBlockLimit() { + return this.pruneBlockLimit; + } + + public long getAtStatesPruneInterval() { + return this.atStatesPruneInterval; + } + + public int getAtStatesPruneBatchSize() { + return this.atStatesPruneBatchSize; + } + + public long getBlockPruneInterval() { + return this.blockPruneInterval; + } + + public int getBlockPruneBatchSize() { + return this.blockPruneBatchSize; + } + + public boolean isNamesIntegrityCheckEnabled() { + return this.namesIntegrityCheckEnabled; + } + + + public boolean isArchiveEnabled() { + if (this.topOnly) { + return false; + } + return this.archiveEnabled; + } + + public long getArchiveInterval() { + return this.archiveInterval; + } + + + public boolean getBootstrap() { + return this.bootstrap; + } + + + public boolean isQdnEnabled() { + return this.qdnEnabled; + } + + public String getDataPath() { + return this.dataPath; + } + + public String getTempDataPath() { + if (this.tempDataPath != null) { + return this.tempDataPath; + } + // Default the temp path to a "_temp" folder inside the data directory + return Paths.get(this.getDataPath(), "_temp").toString(); + } + + public StoragePolicy getStoragePolicy() { + return StoragePolicy.valueOf(this.storagePolicy); + } + + public boolean isRelayModeEnabled() { + return this.relayModeEnabled; + } + + public boolean isDirectDataRetrievalEnabled() { + return this.directDataRetrievalEnabled; + } + + public boolean isOriginalCopyIndicatorFileEnabled() { + return this.originalCopyIndicatorFileEnabled; + } + + public Long getBuiltDataExpiryInterval() { + return this.builtDataExpiryInterval; + } + + public boolean shouldValidateAllDataLayers() { + return this.validateAllDataLayers; + } + + public boolean isPublicDataEnabled() { + return this.publicDataEnabled; + } + + public boolean isPrivateDataEnabled() { + return this.privateDataEnabled; + } + + public Long getMaxStorageCapacity() { + return this.maxStorageCapacity; + } + + public boolean isQDNAuthBypassEnabled() { + return this.qdnAuthBypassEnabled; + } } diff --git a/src/main/java/org/qortal/transaction/AccountFlagsTransaction.java b/src/main/java/org/qortal/transaction/AccountFlagsTransaction.java index 355340b61..4362b1a9a 100644 --- a/src/main/java/org/qortal/transaction/AccountFlagsTransaction.java +++ b/src/main/java/org/qortal/transaction/AccountFlagsTransaction.java @@ -48,6 +48,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.NO_FLAG_PERMISSION; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { Account target = this.getTarget(); diff --git a/src/main/java/org/qortal/transaction/AccountLevelTransaction.java b/src/main/java/org/qortal/transaction/AccountLevelTransaction.java index da986344a..18324c343 100644 --- a/src/main/java/org/qortal/transaction/AccountLevelTransaction.java +++ b/src/main/java/org/qortal/transaction/AccountLevelTransaction.java @@ -49,6 +49,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.NO_FLAG_PERMISSION; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { Account target = getTarget(); diff --git a/src/main/java/org/qortal/transaction/AddGroupAdminTransaction.java b/src/main/java/org/qortal/transaction/AddGroupAdminTransaction.java index d62bd4514..15dc51bfc 100644 --- a/src/main/java/org/qortal/transaction/AddGroupAdminTransaction.java +++ b/src/main/java/org/qortal/transaction/AddGroupAdminTransaction.java @@ -84,6 +84,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Update Group adminship @@ -98,4 +103,4 @@ public void orphan() throws DataException { group.unpromoteToAdmin(this.addGroupAdminTransactionData); } -} \ No newline at end of file +} diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 04ecc09f2..ddb58b0e8 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -1,15 +1,30 @@ package org.qortal.transaction; +import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import org.qortal.account.Account; +import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; +import org.qortal.crypto.Crypto; +import org.qortal.crypto.MemoryPoW; import org.qortal.data.PaymentData; +import org.qortal.data.naming.NameData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; +import org.qortal.network.Network; +import org.qortal.network.message.ArbitrarySignaturesMessage; +import org.qortal.network.message.Message; import org.qortal.payment.Payment; import org.qortal.repository.DataException; import org.qortal.repository.Repository; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.ArbitraryTransactionTransformer; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.ArbitraryTransactionUtils; public class ArbitraryTransaction extends Transaction { @@ -18,6 +33,10 @@ public class ArbitraryTransaction extends Transaction { // Other useful constants public static final int MAX_DATA_SIZE = 4000; + public static final int MAX_METADATA_LENGTH = 32; + public static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; + public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes + public static final int MAX_IDENTIFIER_LENGTH = 64; // Constructors @@ -42,17 +61,148 @@ public Account getSender() { // Processing + public void computeNonce() throws DataException { + byte[] transactionBytes; + + try { + transactionBytes = TransactionTransformer.toBytesForSigning(this.transactionData); + } catch (TransformationException e) { + throw new RuntimeException("Unable to transform transaction to byte array for verification", e); + } + + // Clear nonce from transactionBytes + ArbitraryTransactionTransformer.clearNonce(transactionBytes); + + // Calculate nonce + int difficulty = ArbitraryDataManager.getInstance().getPowDifficulty(); + this.arbitraryTransactionData.setNonce(MemoryPoW.compute2(transactionBytes, POW_BUFFER_SIZE, difficulty)); + } + + @Override + public ValidationResult isFeeValid() throws DataException { + if (this.transactionData.getFee() < 0) + return ValidationResult.NEGATIVE_FEE; + + return ValidationResult.OK; + } + + @Override + public boolean hasValidReference() throws DataException { + // We shouldn't really get this far, but just in case: + if (this.arbitraryTransactionData.getReference() == null) { + return false; + } + + // If the account current doesn't have a last reference, and the fee is 0, we will allow any value. + // This ensures that the first transaction for an account will be valid whilst still validating + // the last reference from the second transaction onwards. By checking for a zero fee, we ensure + // standard last reference validation when fee > 0. + Account creator = getCreator(); + Long fee = this.arbitraryTransactionData.getFee(); + if (creator.getLastReference() == null && fee == 0) { + return true; + } + + return super.hasValidReference(); + } + @Override public ValidationResult isValid() throws DataException { + // Check that some data - or a data hash - has been supplied + if (arbitraryTransactionData.getData() == null) { + return ValidationResult.INVALID_DATA_LENGTH; + } + // Check data length - if (arbitraryTransactionData.getData().length < 1 || arbitraryTransactionData.getData().length > MAX_DATA_SIZE) + if (arbitraryTransactionData.getData().length < 1 || arbitraryTransactionData.getData().length > MAX_DATA_SIZE) { return ValidationResult.INVALID_DATA_LENGTH; + } + + // Check hashes and metadata + if (arbitraryTransactionData.getDataType() == ArbitraryTransactionData.DataType.DATA_HASH) { + // Check length of data hash + if (arbitraryTransactionData.getData().length != HASH_LENGTH) { + return ValidationResult.INVALID_DATA_LENGTH; + } + + // Version 5+ + if (arbitraryTransactionData.getVersion() >= 5) { + byte[] metadata = arbitraryTransactionData.getMetadataHash(); + + // Check maximum length of metadata hash + if (metadata != null && metadata.length > MAX_METADATA_LENGTH) { + return ValidationResult.INVALID_DATA_LENGTH; + } + } + } + + // Check raw data + if (arbitraryTransactionData.getDataType() == ArbitraryTransactionData.DataType.RAW_DATA) { + // Version 5+ + if (arbitraryTransactionData.getVersion() >= 5) { + // Check reported length of the raw data + // We should not download the raw data, so validation of that will be performed later + if (arbitraryTransactionData.getSize() > ArbitraryDataFile.MAX_FILE_SIZE) { + return ValidationResult.INVALID_DATA_LENGTH; + } + } + } + + // Check name if one has been included + if (arbitraryTransactionData.getName() != null) { + NameData nameData = this.repository.getNameRepository().fromName(arbitraryTransactionData.getName()); + + // Check the name is registered + if (nameData == null) { + return ValidationResult.NAME_DOES_NOT_EXIST; + } + + // Check that the transaction signer owns the name + if (!Objects.equals(this.getCreator().getAddress(), nameData.getOwner())) { + return ValidationResult.INVALID_NAME_OWNER; + } + } // Wrap and delegate final payment validity checks to Payment class return new Payment(this.repository).isValid(arbitraryTransactionData.getSenderPublicKey(), arbitraryTransactionData.getPayments(), arbitraryTransactionData.getFee()); } + @Override + public boolean isSignatureValid() { + byte[] signature = this.transactionData.getSignature(); + if (signature == null) { + return false; + } + + byte[] transactionBytes; + + try { + transactionBytes = ArbitraryTransactionTransformer.toBytesForSigning(this.transactionData); + } catch (TransformationException e) { + throw new RuntimeException("Unable to transform transaction to byte array for verification", e); + } + + if (!Crypto.verify(this.transactionData.getCreatorPublicKey(), signature, transactionBytes)) { + return false; + } + + // Nonce wasn't added until version 5+ + if (arbitraryTransactionData.getVersion() >= 5) { + + int nonce = arbitraryTransactionData.getNonce(); + + // Clear nonce from transactionBytes + ArbitraryTransactionTransformer.clearNonce(transactionBytes); + + // Check nonce + int difficulty = ArbitraryDataManager.getInstance().getPowDifficulty(); + return MemoryPoW.verify2(transactionBytes, POW_BUFFER_SIZE, difficulty, nonce); + } + + return true; + } + @Override public ValidationResult isProcessable() throws DataException { // Wrap and delegate final payment processable checks to Payment class @@ -60,6 +210,35 @@ public ValidationResult isProcessable() throws DataException { arbitraryTransactionData.getFee()); } + @Override + protected void onImportAsUnconfirmed() throws DataException { + // We may need to move files from the misc_ folder + ArbitraryTransactionUtils.checkAndRelocateMiscFiles(arbitraryTransactionData); + + // If the data is local, we need to perform a few actions + if (isDataLocal()) { + + // We have the data for this transaction, so invalidate the cache + if (arbitraryTransactionData.getName() != null) { + ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData); + } + + // We also need to broadcast to the network that we are now hosting files for this transaction, + // but only if these files are in accordance with our storage policy + if (ArbitraryDataStorageManager.getInstance().canStoreData(arbitraryTransactionData)) { + // Use a null peer address to indicate our own + byte[] signature = arbitraryTransactionData.getSignature(); + Message arbitrarySignatureMessage = new ArbitrarySignaturesMessage(null, 0, Arrays.asList(signature)); + Network.getInstance().broadcast(broadcastPeer -> arbitrarySignatureMessage); + } + } + } + + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Wrap and delegate payment processing to Payment class. @@ -95,10 +274,9 @@ public boolean isDataLocal() throws DataException { /** Returns arbitrary data payload, fetching from network if needed. Can block for a while! */ public byte[] fetchData() throws DataException { // If local, read from file - if (isDataLocal()) + if (isDataLocal()) { return this.repository.getArbitraryRepository().fetchData(this.transactionData.getSignature()); - - // TODO If not local, attempt to fetch via network? + } return null; } diff --git a/src/main/java/org/qortal/transaction/AtTransaction.java b/src/main/java/org/qortal/transaction/AtTransaction.java index a7e72b2a2..c570bb653 100644 --- a/src/main/java/org/qortal/transaction/AtTransaction.java +++ b/src/main/java/org/qortal/transaction/AtTransaction.java @@ -80,6 +80,11 @@ public boolean hasValidReference() throws DataException { return Arrays.equals(atAccount.getLastReference(), atTransactionData.getReference()); } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public ValidationResult isValid() throws DataException { // Check recipient address is valid diff --git a/src/main/java/org/qortal/transaction/BuyNameTransaction.java b/src/main/java/org/qortal/transaction/BuyNameTransaction.java index ad3e0c8d7..c4e5f29ce 100644 --- a/src/main/java/org/qortal/transaction/BuyNameTransaction.java +++ b/src/main/java/org/qortal/transaction/BuyNameTransaction.java @@ -6,6 +6,7 @@ import org.qortal.account.Account; import org.qortal.asset.Asset; import org.qortal.block.BlockChain; +import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.crypto.Crypto; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.BuyNameTransactionData; @@ -98,6 +99,17 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData; + + // Rebuild this name in the Names table from the transaction history + // This is necessary because in some rare cases names can be missing from the Names table after registration + // but we have been unable to reproduce the issue and track down the root cause + NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck(); + namesDatabaseIntegrityCheck.rebuildName(buyNameTransactionData.getName(), this.repository); + } + @Override public void process() throws DataException { // Buy Name diff --git a/src/main/java/org/qortal/transaction/CancelAssetOrderTransaction.java b/src/main/java/org/qortal/transaction/CancelAssetOrderTransaction.java index b8b70dde5..955f62f43 100644 --- a/src/main/java/org/qortal/transaction/CancelAssetOrderTransaction.java +++ b/src/main/java/org/qortal/transaction/CancelAssetOrderTransaction.java @@ -62,6 +62,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Mark Order as completed so no more trades can happen diff --git a/src/main/java/org/qortal/transaction/CancelGroupBanTransaction.java b/src/main/java/org/qortal/transaction/CancelGroupBanTransaction.java index e01be7bed..483dfc6fb 100644 --- a/src/main/java/org/qortal/transaction/CancelGroupBanTransaction.java +++ b/src/main/java/org/qortal/transaction/CancelGroupBanTransaction.java @@ -83,6 +83,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Update Group Membership diff --git a/src/main/java/org/qortal/transaction/CancelGroupInviteTransaction.java b/src/main/java/org/qortal/transaction/CancelGroupInviteTransaction.java index ea2282150..800f24447 100644 --- a/src/main/java/org/qortal/transaction/CancelGroupInviteTransaction.java +++ b/src/main/java/org/qortal/transaction/CancelGroupInviteTransaction.java @@ -83,6 +83,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Update Group Membership diff --git a/src/main/java/org/qortal/transaction/CancelSellNameTransaction.java b/src/main/java/org/qortal/transaction/CancelSellNameTransaction.java index f241db476..788492a9a 100644 --- a/src/main/java/org/qortal/transaction/CancelSellNameTransaction.java +++ b/src/main/java/org/qortal/transaction/CancelSellNameTransaction.java @@ -79,6 +79,11 @@ public ValidationResult isValid() throws DataException { } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Update Name diff --git a/src/main/java/org/qortal/transaction/ChatTransaction.java b/src/main/java/org/qortal/transaction/ChatTransaction.java index d3eec9f75..a486d408c 100644 --- a/src/main/java/org/qortal/transaction/ChatTransaction.java +++ b/src/main/java/org/qortal/transaction/ChatTransaction.java @@ -8,9 +8,11 @@ import org.qortal.asset.Asset; import org.qortal.crypto.Crypto; import org.qortal.crypto.MemoryPoW; +import org.qortal.data.naming.NameData; import org.qortal.data.transaction.ChatTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.group.Group; +import org.qortal.list.ResourceListManager; import org.qortal.repository.DataException; import org.qortal.repository.GroupRepository; import org.qortal.repository.Repository; @@ -134,14 +136,37 @@ public boolean hasValidReference() throws DataException { return true; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public ValidationResult isValid() throws DataException { // Nonce checking is done via isSignatureValid() as that method is only called once per import + // Check for blocked author by address + ResourceListManager listManager = ResourceListManager.getInstance(); + if (listManager.listContains("blockedAddresses", this.chatTransactionData.getSender(), true)) { + return ValidationResult.ADDRESS_BLOCKED; + } + + // Check for blocked author by registered name + List names = this.repository.getNameRepository().getNamesByOwner(this.chatTransactionData.getSender()); + if (names != null && names.size() > 0) { + for (NameData nameData : names) { + if (nameData != null && nameData.getName() != null) { + if (listManager.listContains("blockedNames", nameData.getName(), false)) { + return ValidationResult.NAME_BLOCKED; + } + } + } + } + // If we exist in the repository then we've been imported as unconfirmed, // but we don't want to make it into a block, so return fake non-OK result. if (this.repository.getTransactionRepository().exists(this.chatTransactionData.getSignature())) - return ValidationResult.CHAT; + return ValidationResult.INVALID_BUT_OK; // If we have a recipient, check it is a valid address String recipientAddress = chatTransactionData.getRecipient(); @@ -188,6 +213,16 @@ public boolean isSignatureValid() { return MemoryPoW.verify2(transactionBytes, POW_BUFFER_SIZE, difficulty, nonce); } + /** + * Ensure there's at least a skeleton account so people + * can retrieve sender's public key using address, even if all their messages + * expire. + */ + @Override + protected void onImportAsUnconfirmed() throws DataException { + this.getCreator().ensureAccount(); + } + @Override public void process() throws DataException { throw new DataException("CHAT transactions should never be processed"); diff --git a/src/main/java/org/qortal/transaction/CreateAssetOrderTransaction.java b/src/main/java/org/qortal/transaction/CreateAssetOrderTransaction.java index 36cccf42e..24e57a4ee 100644 --- a/src/main/java/org/qortal/transaction/CreateAssetOrderTransaction.java +++ b/src/main/java/org/qortal/transaction/CreateAssetOrderTransaction.java @@ -135,6 +135,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Order Id is transaction's signature diff --git a/src/main/java/org/qortal/transaction/CreateGroupTransaction.java b/src/main/java/org/qortal/transaction/CreateGroupTransaction.java index 7ed61684d..6f4a36346 100644 --- a/src/main/java/org/qortal/transaction/CreateGroupTransaction.java +++ b/src/main/java/org/qortal/transaction/CreateGroupTransaction.java @@ -92,6 +92,11 @@ public ValidationResult isProcessable() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Create Group diff --git a/src/main/java/org/qortal/transaction/CreatePollTransaction.java b/src/main/java/org/qortal/transaction/CreatePollTransaction.java index 4c4b3a0af..a56322a71 100644 --- a/src/main/java/org/qortal/transaction/CreatePollTransaction.java +++ b/src/main/java/org/qortal/transaction/CreatePollTransaction.java @@ -106,6 +106,11 @@ public ValidationResult isProcessable() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Publish poll to allow voting diff --git a/src/main/java/org/qortal/transaction/DeployAtTransaction.java b/src/main/java/org/qortal/transaction/DeployAtTransaction.java index 46ad9e3e0..f3024b57b 100644 --- a/src/main/java/org/qortal/transaction/DeployAtTransaction.java +++ b/src/main/java/org/qortal/transaction/DeployAtTransaction.java @@ -26,45 +26,47 @@ public class DeployAtTransaction extends Transaction { // Properties - private DeployAtTransactionData deployATTransactionData; + private DeployAtTransactionData deployAtTransactionData; // Other useful constants public static final int MAX_NAME_SIZE = 200; public static final int MAX_DESCRIPTION_SIZE = 2000; public static final int MAX_AT_TYPE_SIZE = 200; public static final int MAX_TAGS_SIZE = 200; - public static final int MAX_CREATION_BYTES_SIZE = 100_000; + public static final int MAX_CREATION_BYTES_SIZE = 4096; + public static final int MAX_CODE_BYTES_LENGTH = 1024; + public static final int MAX_AT_STATE_LENGTH = 1024; // Constructors public DeployAtTransaction(Repository repository, TransactionData transactionData) { super(repository, transactionData); - this.deployATTransactionData = (DeployAtTransactionData) this.transactionData; + this.deployAtTransactionData = (DeployAtTransactionData) this.transactionData; } // More information @Override public List getRecipientAddresses() throws DataException { - return Collections.singletonList(this.deployATTransactionData.getAtAddress()); + return Collections.singletonList(this.deployAtTransactionData.getAtAddress()); } /** Returns AT version from the header bytes */ private short getVersion() { - byte[] creationBytes = deployATTransactionData.getCreationBytes(); + byte[] creationBytes = deployAtTransactionData.getCreationBytes(); return (short) ((creationBytes[0] << 8) | (creationBytes[1] & 0xff)); // Big-endian } /** Make sure deployATTransactionData has an ATAddress */ - private void ensureATAddress() throws DataException { - if (this.deployATTransactionData.getAtAddress() != null) + public static void ensureATAddress(DeployAtTransactionData deployAtTransactionData) throws DataException { + if (deployAtTransactionData.getAtAddress() != null) return; // Use transaction transformer try { - String atAddress = Crypto.toATAddress(TransactionTransformer.toBytesForSigning(this.deployATTransactionData)); - this.deployATTransactionData.setAtAddress(atAddress); + String atAddress = Crypto.toATAddress(TransactionTransformer.toBytesForSigning(deployAtTransactionData)); + deployAtTransactionData.setAtAddress(atAddress); } catch (TransformationException e) { throw new DataException("Unable to generate AT address"); } @@ -73,9 +75,9 @@ private void ensureATAddress() throws DataException { // Navigation public Account getATAccount() throws DataException { - ensureATAddress(); + ensureATAddress(this.deployAtTransactionData); - return new Account(this.repository, this.deployATTransactionData.getAtAddress()); + return new Account(this.repository, this.deployAtTransactionData.getAtAddress()); } // Processing @@ -83,30 +85,30 @@ public Account getATAccount() throws DataException { @Override public ValidationResult isValid() throws DataException { // Check name size bounds - int nameLength = Utf8.encodedLength(this.deployATTransactionData.getName()); + int nameLength = Utf8.encodedLength(this.deployAtTransactionData.getName()); if (nameLength < 1 || nameLength > MAX_NAME_SIZE) return ValidationResult.INVALID_NAME_LENGTH; // Check description size bounds - int descriptionlength = Utf8.encodedLength(this.deployATTransactionData.getDescription()); + int descriptionlength = Utf8.encodedLength(this.deployAtTransactionData.getDescription()); if (descriptionlength < 1 || descriptionlength > MAX_DESCRIPTION_SIZE) return ValidationResult.INVALID_DESCRIPTION_LENGTH; // Check AT-type size bounds - int atTypeLength = Utf8.encodedLength(this.deployATTransactionData.getAtType()); + int atTypeLength = Utf8.encodedLength(this.deployAtTransactionData.getAtType()); if (atTypeLength < 1 || atTypeLength > MAX_AT_TYPE_SIZE) return ValidationResult.INVALID_AT_TYPE_LENGTH; // Check tags size bounds - int tagsLength = Utf8.encodedLength(this.deployATTransactionData.getTags()); + int tagsLength = Utf8.encodedLength(this.deployAtTransactionData.getTags()); if (tagsLength < 1 || tagsLength > MAX_TAGS_SIZE) return ValidationResult.INVALID_TAGS_LENGTH; // Check amount is positive - if (this.deployATTransactionData.getAmount() <= 0) + if (this.deployAtTransactionData.getAmount() <= 0) return ValidationResult.NEGATIVE_AMOUNT; - long assetId = this.deployATTransactionData.getAssetId(); + long assetId = this.deployAtTransactionData.getAssetId(); AssetData assetData = this.repository.getAssetRepository().fromAssetId(assetId); // Check asset even exists if (assetData == null) @@ -117,7 +119,7 @@ public ValidationResult isValid() throws DataException { return ValidationResult.ASSET_NOT_SPENDABLE; // Check asset amount is integer if asset is not divisible - if (!assetData.isDivisible() && this.deployATTransactionData.getAmount() % Amounts.MULTIPLIER != 0) + if (!assetData.isDivisible() && this.deployAtTransactionData.getAmount() % Amounts.MULTIPLIER != 0) return ValidationResult.INVALID_AMOUNT; Account creator = this.getCreator(); @@ -125,15 +127,15 @@ public ValidationResult isValid() throws DataException { // Check creator has enough funds if (assetId == Asset.QORT) { // Simple case: amount and fee both in QORT - long minimumBalance = this.deployATTransactionData.getFee() + this.deployATTransactionData.getAmount(); + long minimumBalance = this.deployAtTransactionData.getFee() + this.deployAtTransactionData.getAmount(); if (creator.getConfirmedBalance(Asset.QORT) < minimumBalance) return ValidationResult.NO_BALANCE; } else { - if (creator.getConfirmedBalance(Asset.QORT) < this.deployATTransactionData.getFee()) + if (creator.getConfirmedBalance(Asset.QORT) < this.deployAtTransactionData.getFee()) return ValidationResult.NO_BALANCE; - if (creator.getConfirmedBalance(assetId) < this.deployATTransactionData.getAmount()) + if (creator.getConfirmedBalance(assetId) < this.deployAtTransactionData.getAmount()) return ValidationResult.NO_BALANCE; } @@ -142,12 +144,12 @@ public ValidationResult isValid() throws DataException { return ValidationResult.INVALID_CREATION_BYTES; // Check creation bytes are valid (for v2+) - this.ensureATAddress(); + ensureATAddress(this.deployAtTransactionData); // Just enough AT data to allow API to query initial balances, etc. - String atAddress = this.deployATTransactionData.getAtAddress(); - byte[] creatorPublicKey = this.deployATTransactionData.getCreatorPublicKey(); - long creation = this.deployATTransactionData.getTimestamp(); + String atAddress = this.deployAtTransactionData.getAtAddress(); + byte[] creatorPublicKey = this.deployAtTransactionData.getCreatorPublicKey(); + long creation = this.deployAtTransactionData.getTimestamp(); ATData skeletonAtData = new ATData(atAddress, creatorPublicKey, creation, assetId); int height = this.repository.getBlockRepository().getBlockchainHeight() + 1; @@ -157,7 +159,15 @@ public ValidationResult isValid() throws DataException { QortalAtLoggerFactory loggerFactory = QortalAtLoggerFactory.getInstance(); try { - new MachineState(api, loggerFactory, this.deployATTransactionData.getCreationBytes()); + MachineState state = new MachineState(api, loggerFactory, this.deployAtTransactionData.getCreationBytes()); + + byte[] codeBytes = state.getCodeBytes(); + if (codeBytes == null || codeBytes.length > MAX_CODE_BYTES_LENGTH) + return ValidationResult.INVALID_CREATION_BYTES; + + byte[] atStateBytes = state.toBytes(); + if (atStateBytes == null || atStateBytes.length > MAX_AT_STATE_LENGTH) + return ValidationResult.INVALID_CREATION_BYTES; } catch (IllegalArgumentException e) { // Not valid return ValidationResult.INVALID_CREATION_BYTES; @@ -169,66 +179,71 @@ public ValidationResult isValid() throws DataException { @Override public ValidationResult isProcessable() throws DataException { Account creator = getCreator(); - long assetId = this.deployATTransactionData.getAssetId(); + long assetId = this.deployAtTransactionData.getAssetId(); // Check creator has enough funds if (assetId == Asset.QORT) { // Simple case: amount and fee both in QORT - long minimumBalance = this.deployATTransactionData.getFee() + this.deployATTransactionData.getAmount(); + long minimumBalance = this.deployAtTransactionData.getFee() + this.deployAtTransactionData.getAmount(); if (creator.getConfirmedBalance(Asset.QORT) < minimumBalance) return ValidationResult.NO_BALANCE; } else { - if (creator.getConfirmedBalance(Asset.QORT) < this.deployATTransactionData.getFee()) + if (creator.getConfirmedBalance(Asset.QORT) < this.deployAtTransactionData.getFee()) return ValidationResult.NO_BALANCE; - if (creator.getConfirmedBalance(assetId) < this.deployATTransactionData.getAmount()) + if (creator.getConfirmedBalance(assetId) < this.deployAtTransactionData.getAmount()) return ValidationResult.NO_BALANCE; } // Check AT doesn't already exist - if (this.repository.getATRepository().exists(this.deployATTransactionData.getAtAddress())) + if (this.repository.getATRepository().exists(this.deployAtTransactionData.getAtAddress())) return ValidationResult.AT_ALREADY_EXISTS; return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { - this.ensureATAddress(); + ensureATAddress(this.deployAtTransactionData); // Deploy AT, saving into repository - AT at = new AT(this.repository, this.deployATTransactionData); + AT at = new AT(this.repository, this.deployAtTransactionData); at.deploy(); - long assetId = this.deployATTransactionData.getAssetId(); + long assetId = this.deployAtTransactionData.getAssetId(); // Update creator's balance regarding initial payment to AT Account creator = getCreator(); - creator.modifyAssetBalance(assetId, - this.deployATTransactionData.getAmount()); + creator.modifyAssetBalance(assetId, - this.deployAtTransactionData.getAmount()); // Update AT's reference, which also creates AT account Account atAccount = this.getATAccount(); - atAccount.setLastReference(this.deployATTransactionData.getSignature()); + atAccount.setLastReference(this.deployAtTransactionData.getSignature()); // Update AT's balance - atAccount.setConfirmedBalance(assetId, this.deployATTransactionData.getAmount()); + atAccount.setConfirmedBalance(assetId, this.deployAtTransactionData.getAmount()); } @Override public void orphan() throws DataException { // Delete AT from repository - AT at = new AT(this.repository, this.deployATTransactionData); + AT at = new AT(this.repository, this.deployAtTransactionData); at.undeploy(); - long assetId = this.deployATTransactionData.getAssetId(); + long assetId = this.deployAtTransactionData.getAssetId(); // Update creator's balance regarding initial payment to AT Account creator = getCreator(); - creator.modifyAssetBalance(assetId, this.deployATTransactionData.getAmount()); + creator.modifyAssetBalance(assetId, this.deployAtTransactionData.getAmount()); // Delete AT's account (and hence its balance) - this.repository.getAccountRepository().delete(this.deployATTransactionData.getAtAddress()); + this.repository.getAccountRepository().delete(this.deployAtTransactionData.getAtAddress()); } } diff --git a/src/main/java/org/qortal/transaction/GenesisTransaction.java b/src/main/java/org/qortal/transaction/GenesisTransaction.java index 067ff1838..74a84a7dc 100644 --- a/src/main/java/org/qortal/transaction/GenesisTransaction.java +++ b/src/main/java/org/qortal/transaction/GenesisTransaction.java @@ -100,6 +100,11 @@ public ValidationResult isValid() { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { Account recipient = new Account(repository, this.genesisTransactionData.getRecipient()); diff --git a/src/main/java/org/qortal/transaction/GroupApprovalTransaction.java b/src/main/java/org/qortal/transaction/GroupApprovalTransaction.java index d5cf66f7e..1c8bb7094 100644 --- a/src/main/java/org/qortal/transaction/GroupApprovalTransaction.java +++ b/src/main/java/org/qortal/transaction/GroupApprovalTransaction.java @@ -66,6 +66,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Find previous approval decision (if any) by this admin for pending transaction diff --git a/src/main/java/org/qortal/transaction/GroupBanTransaction.java b/src/main/java/org/qortal/transaction/GroupBanTransaction.java index d3458ebe3..c9a6c3077 100644 --- a/src/main/java/org/qortal/transaction/GroupBanTransaction.java +++ b/src/main/java/org/qortal/transaction/GroupBanTransaction.java @@ -87,6 +87,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Update Group Membership diff --git a/src/main/java/org/qortal/transaction/GroupInviteTransaction.java b/src/main/java/org/qortal/transaction/GroupInviteTransaction.java index a66f7584c..f3b08f599 100644 --- a/src/main/java/org/qortal/transaction/GroupInviteTransaction.java +++ b/src/main/java/org/qortal/transaction/GroupInviteTransaction.java @@ -88,6 +88,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Update Group Membership diff --git a/src/main/java/org/qortal/transaction/GroupKickTransaction.java b/src/main/java/org/qortal/transaction/GroupKickTransaction.java index d9be8161d..84de3a594 100644 --- a/src/main/java/org/qortal/transaction/GroupKickTransaction.java +++ b/src/main/java/org/qortal/transaction/GroupKickTransaction.java @@ -89,6 +89,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Update Group Membership diff --git a/src/main/java/org/qortal/transaction/IssueAssetTransaction.java b/src/main/java/org/qortal/transaction/IssueAssetTransaction.java index e9422dcdd..524289635 100644 --- a/src/main/java/org/qortal/transaction/IssueAssetTransaction.java +++ b/src/main/java/org/qortal/transaction/IssueAssetTransaction.java @@ -92,6 +92,11 @@ public ValidationResult isProcessable() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Issue asset diff --git a/src/main/java/org/qortal/transaction/JoinGroupTransaction.java b/src/main/java/org/qortal/transaction/JoinGroupTransaction.java index ed69ed4e8..bc62c629a 100644 --- a/src/main/java/org/qortal/transaction/JoinGroupTransaction.java +++ b/src/main/java/org/qortal/transaction/JoinGroupTransaction.java @@ -67,6 +67,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Update Group Membership diff --git a/src/main/java/org/qortal/transaction/LeaveGroupTransaction.java b/src/main/java/org/qortal/transaction/LeaveGroupTransaction.java index ad31e565b..1e8f8c6cf 100644 --- a/src/main/java/org/qortal/transaction/LeaveGroupTransaction.java +++ b/src/main/java/org/qortal/transaction/LeaveGroupTransaction.java @@ -67,6 +67,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Update Group Membership diff --git a/src/main/java/org/qortal/transaction/MessageTransaction.java b/src/main/java/org/qortal/transaction/MessageTransaction.java index ef6e6c76d..d02b6fdd2 100644 --- a/src/main/java/org/qortal/transaction/MessageTransaction.java +++ b/src/main/java/org/qortal/transaction/MessageTransaction.java @@ -239,6 +239,11 @@ public ValidationResult isProcessable() throws DataException { getPaymentData(), this.messageTransactionData.getFee(), true); } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // If we have no amount then there's nothing to do diff --git a/src/main/java/org/qortal/transaction/MultiPaymentTransaction.java b/src/main/java/org/qortal/transaction/MultiPaymentTransaction.java index 4c3f75dce..34cd01475 100644 --- a/src/main/java/org/qortal/transaction/MultiPaymentTransaction.java +++ b/src/main/java/org/qortal/transaction/MultiPaymentTransaction.java @@ -67,6 +67,11 @@ public ValidationResult isProcessable() throws DataException { return new Payment(this.repository).isProcessable(this.multiPaymentTransactionData.getSenderPublicKey(), payments, this.multiPaymentTransactionData.getFee()); } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Wrap and delegate payment processing to Payment class. diff --git a/src/main/java/org/qortal/transaction/PaymentTransaction.java b/src/main/java/org/qortal/transaction/PaymentTransaction.java index f6caaef5a..4869db76f 100644 --- a/src/main/java/org/qortal/transaction/PaymentTransaction.java +++ b/src/main/java/org/qortal/transaction/PaymentTransaction.java @@ -61,6 +61,11 @@ public ValidationResult isProcessable() throws DataException { return new Payment(this.repository).isProcessable(this.paymentTransactionData.getSenderPublicKey(), getPaymentData(), this.paymentTransactionData.getFee()); } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Wrap and delegate payment processing to Payment class. diff --git a/src/main/java/org/qortal/transaction/PresenceTransaction.java b/src/main/java/org/qortal/transaction/PresenceTransaction.java new file mode 100644 index 000000000..0d28d3823 --- /dev/null +++ b/src/main/java/org/qortal/transaction/PresenceTransaction.java @@ -0,0 +1,261 @@ +package org.qortal.transaction; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.account.Account; +import org.qortal.controller.Controller; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crypto.Crypto; +import org.qortal.crypto.MemoryPoW; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.PresenceTransactionTransformer; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.ByteArray; + +import com.google.common.primitives.Longs; + +public class PresenceTransaction extends Transaction { + + private static final Logger LOGGER = LogManager.getLogger(PresenceTransaction.class); + + // Properties + private PresenceTransactionData presenceTransactionData; + + // Other useful constants + public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes + public static final int POW_DIFFICULTY = 8; // leading zero bits + + public enum PresenceType { + REWARD_SHARE(0) { + @Override + public long getLifetime() { + return Controller.ONLINE_TIMESTAMP_MODULUS; + } + }, + TRADE_BOT(1) { + @Override + public long getLifetime() { + return 30 * 60 * 1000L; // 30 minutes in milliseconds + } + }; + + public final int value; + private static final Map map = stream(PresenceType.values()).collect(toMap(type -> type.value, type -> type)); + + PresenceType(int value) { + this.value = value; + } + + public abstract long getLifetime(); + + public static PresenceType valueOf(int value) { + return map.get(value); + } + + /** Returns PresenceType with matching name or null (instead of throwing IllegalArgumentException). */ + public static PresenceType fromString(String name) { + try { + return PresenceType.valueOf(name); + } catch (IllegalArgumentException e) { + return null; + } + } + } + + // Constructors + + public PresenceTransaction(Repository repository, TransactionData transactionData) { + super(repository, transactionData); + + this.presenceTransactionData = (PresenceTransactionData) this.transactionData; + } + + // More information + + @Override + public long getDeadline() { + return this.transactionData.getTimestamp() + this.presenceTransactionData.getPresenceType().getLifetime(); + } + + @Override + public List getRecipientAddresses() throws DataException { + return Collections.emptyList(); + } + + // Navigation + + public Account getSender() { + return this.getCreator(); + } + + // Processing + + public void computeNonce() throws DataException { + byte[] transactionBytes; + + try { + transactionBytes = TransactionTransformer.toBytesForSigning(this.transactionData); + } catch (TransformationException e) { + throw new RuntimeException("Unable to transform transaction to byte array for verification", e); + } + + // Clear nonce from transactionBytes + PresenceTransactionTransformer.clearNonce(transactionBytes); + + // Calculate nonce + this.presenceTransactionData.setNonce(MemoryPoW.compute2(transactionBytes, POW_BUFFER_SIZE, POW_DIFFICULTY)); + } + + /** + * Returns whether PRESENCE transaction has valid txGroupId. + *

+ * We insist on NO_GROUP. + */ + @Override + protected boolean isValidTxGroupId() throws DataException { + int txGroupId = this.transactionData.getTxGroupId(); + + return txGroupId == Group.NO_GROUP; + } + + @Override + public ValidationResult isFeeValid() throws DataException { + if (this.transactionData.getFee() < 0) + return ValidationResult.NEGATIVE_FEE; + + return ValidationResult.OK; + } + + @Override + public boolean hasValidReference() throws DataException { + return true; + } + + @Override + public void preProcess() throws DataException { + // Nothing to do + } + + @Override + public ValidationResult isValid() throws DataException { + // Nonce checking is done via isSignatureValid() as that method is only called once per import + + // If we exist in the repository then we've been imported as unconfirmed, + // but we don't want to make it into a block, so return fake non-OK result. + if (this.repository.getTransactionRepository().exists(this.presenceTransactionData.getSignature())) + return ValidationResult.INVALID_BUT_OK; + + // We only support TRADE_BOT-type PRESENCE at this time + if (PresenceType.TRADE_BOT != this.presenceTransactionData.getPresenceType()) + return ValidationResult.NOT_YET_RELEASED; + + // Check timestamp signature + byte[] timestampSignature = this.presenceTransactionData.getTimestampSignature(); + byte[] timestampBytes = Longs.toByteArray(this.presenceTransactionData.getTimestamp()); + if (!Crypto.verify(this.transactionData.getCreatorPublicKey(), timestampSignature, timestampBytes)) + return ValidationResult.INVALID_TIMESTAMP_SIGNATURE; + + Map> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap(); + Set codeHashes = acctSuppliersByCodeHash.keySet(); + boolean isExecutable = true; + + List atsData = repository.getATRepository().getAllATsByFunctionality(codeHashes, isExecutable); + + // Convert signer's public key to address form + String signerAddress = Crypto.toAddress(this.transactionData.getCreatorPublicKey()); + + for (ATData atData : atsData) { + ByteArray atCodeHash = new ByteArray(atData.getCodeHash()); + Supplier acctSupplier = acctSuppliersByCodeHash.get(atCodeHash); + if (acctSupplier == null) + continue; + + CrossChainTradeData crossChainTradeData = acctSupplier.get().populateTradeData(repository, atData); + + // OK if signer's public key (in address form) matches Bob's trade public key (in address form) + if (signerAddress.equals(crossChainTradeData.qortalCreatorTradeAddress)) + return ValidationResult.OK; + + // OK if signer's public key (in address form) matches Alice's trade public key (in address form) + if (signerAddress.equals(crossChainTradeData.qortalPartnerAddress)) + return ValidationResult.OK; + } + + return ValidationResult.AT_UNKNOWN; + } + + @Override + public boolean isSignatureValid() { + byte[] signature = this.transactionData.getSignature(); + if (signature == null) + return false; + + byte[] transactionBytes; + + try { + transactionBytes = PresenceTransactionTransformer.toBytesForSigning(this.transactionData); + } catch (TransformationException e) { + throw new RuntimeException("Unable to transform transaction to byte array for verification", e); + } + + if (!Crypto.verify(this.transactionData.getCreatorPublicKey(), signature, transactionBytes)) + return false; + + int nonce = this.presenceTransactionData.getNonce(); + + // Clear nonce from transactionBytes + PresenceTransactionTransformer.clearNonce(transactionBytes); + + // Check nonce + return MemoryPoW.verify2(transactionBytes, POW_BUFFER_SIZE, POW_DIFFICULTY, nonce); + } + + /** + * Remove any PRESENCE transactions by the same signer that have older timestamps. + */ + @Override + protected void onImportAsUnconfirmed() throws DataException { + byte[] creatorPublicKey = this.transactionData.getCreatorPublicKey(); + List creatorsPresenceTransactions = this.repository.getTransactionRepository().getUnconfirmedTransactions(TransactionType.PRESENCE, creatorPublicKey); + + if (creatorsPresenceTransactions.isEmpty()) + return; + + for (TransactionData transactionData : creatorsPresenceTransactions) { + if (transactionData.getTimestamp() >= this.transactionData.getTimestamp()) + continue; + + LOGGER.debug(() -> String.format("Deleting older PRESENCE transaction %s", Base58.encode(transactionData.getSignature()))); + this.repository.getTransactionRepository().delete(transactionData); + } + } + + @Override + public void process() throws DataException { + throw new DataException("PRESENCE transactions should never be processed"); + } + + @Override + public void orphan() throws DataException { + throw new DataException("PRESENCE transactions should never be orphaned"); + } + +} diff --git a/src/main/java/org/qortal/transaction/PublicizeTransaction.java b/src/main/java/org/qortal/transaction/PublicizeTransaction.java index 75cfd2a2b..c03c8283e 100644 --- a/src/main/java/org/qortal/transaction/PublicizeTransaction.java +++ b/src/main/java/org/qortal/transaction/PublicizeTransaction.java @@ -80,6 +80,11 @@ public boolean hasValidReference() throws DataException { return true; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public ValidationResult isValid() throws DataException { // There can be only one diff --git a/src/main/java/org/qortal/transaction/RegisterNameTransaction.java b/src/main/java/org/qortal/transaction/RegisterNameTransaction.java index 66c1fc8bd..1ababa881 100644 --- a/src/main/java/org/qortal/transaction/RegisterNameTransaction.java +++ b/src/main/java/org/qortal/transaction/RegisterNameTransaction.java @@ -6,6 +6,7 @@ import org.qortal.account.Account; import org.qortal.asset.Asset; import org.qortal.block.BlockChain; +import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.crypto.Crypto; import org.qortal.data.transaction.RegisterNameTransactionData; import org.qortal.data.transaction.TransactionData; @@ -36,6 +37,15 @@ public List getRecipientAddresses() throws DataException { return Collections.emptyList(); } + @Override + public long getUnitFee(Long timestamp) { + // Use a higher unit fee after the fee increase timestamp + if (timestamp > BlockChain.getInstance().getNameRegistrationUnitFeeTimestamp()) { + return BlockChain.getInstance().getNameRegistrationUnitFee(); + } + return BlockChain.getInstance().getUnitFee(); + } + // Navigation public Account getRegistrant() { @@ -88,6 +98,17 @@ public ValidationResult isProcessable() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData; + + // Rebuild this name in the Names table from the transaction history + // This is necessary because in some rare cases names can be missing from the Names table after registration + // but we have been unable to reproduce the issue and track down the root cause + NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck(); + namesDatabaseIntegrityCheck.rebuildName(registerNameTransactionData.getName(), this.repository); + } + @Override public void process() throws DataException { // Register Name diff --git a/src/main/java/org/qortal/transaction/RemoveGroupAdminTransaction.java b/src/main/java/org/qortal/transaction/RemoveGroupAdminTransaction.java index 43f1fc8f5..3e5f1e6d0 100644 --- a/src/main/java/org/qortal/transaction/RemoveGroupAdminTransaction.java +++ b/src/main/java/org/qortal/transaction/RemoveGroupAdminTransaction.java @@ -87,6 +87,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Update Group adminship @@ -107,4 +112,4 @@ public void orphan() throws DataException { this.repository.getTransactionRepository().save(this.removeGroupAdminTransactionData); } -} \ No newline at end of file +} diff --git a/src/main/java/org/qortal/transaction/RewardShareTransaction.java b/src/main/java/org/qortal/transaction/RewardShareTransaction.java index c22c53002..be68196d5 100644 --- a/src/main/java/org/qortal/transaction/RewardShareTransaction.java +++ b/src/main/java/org/qortal/transaction/RewardShareTransaction.java @@ -136,7 +136,7 @@ public ValidationResult isValid() throws DataException { // Deleting a non-existent reward-share makes no sense if (isCancellingSharePercent) - return ValidationResult.INVALID_REWARD_SHARE_PERCENT; + return ValidationResult.REWARD_SHARE_UNKNOWN; // Check the minting account hasn't reach maximum number of reward-shares int rewardShareCount = this.repository.getAccountRepository().countRewardShares(creator.getPublicKey()); @@ -159,6 +159,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { PublicKeyAccount mintingAccount = getMintingAccount(); diff --git a/src/main/java/org/qortal/transaction/SellNameTransaction.java b/src/main/java/org/qortal/transaction/SellNameTransaction.java index 81bd9ff7f..c2ab2eb95 100644 --- a/src/main/java/org/qortal/transaction/SellNameTransaction.java +++ b/src/main/java/org/qortal/transaction/SellNameTransaction.java @@ -5,6 +5,7 @@ import org.qortal.account.Account; import org.qortal.asset.Asset; +import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.SellNameTransactionData; import org.qortal.data.transaction.TransactionData; @@ -89,6 +90,17 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData; + + // Rebuild this name in the Names table from the transaction history + // This is necessary because in some rare cases names can be missing from the Names table after registration + // but we have been unable to reproduce the issue and track down the root cause + NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck(); + namesDatabaseIntegrityCheck.rebuildName(sellNameTransactionData.getName(), this.repository); + } + @Override public void process() throws DataException { // Sell Name diff --git a/src/main/java/org/qortal/transaction/SetGroupTransaction.java b/src/main/java/org/qortal/transaction/SetGroupTransaction.java index 084044a7d..48248b697 100644 --- a/src/main/java/org/qortal/transaction/SetGroupTransaction.java +++ b/src/main/java/org/qortal/transaction/SetGroupTransaction.java @@ -56,6 +56,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { Account creator = getCreator(); diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index bdddfb1a2..79a6478b4 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.EnumSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -83,7 +84,8 @@ public enum TransactionType { ENABLE_FORGING(37, false), REWARD_SHARE(38, false), ACCOUNT_LEVEL(39, false), - TRANSFER_PRIVS(40, false); + TRANSFER_PRIVS(40, false), + PRESENCE(41, false); public final int value; public final boolean needsApproval; @@ -225,6 +227,7 @@ public enum ValidationResult { AT_IS_FINISHED(71), NO_FLAG_PERMISSION(72), NOT_MINTING_ACCOUNT(73), + REWARD_SHARE_UNKNOWN(76), INVALID_REWARD_SHARE_PERCENT(77), PUBLIC_KEY_UNKNOWN(78), INVALID_PUBLIC_KEY(79), @@ -243,7 +246,10 @@ public enum ValidationResult { ACCOUNT_ALREADY_EXISTS(92), INVALID_GROUP_BLOCK_DELAY(93), INCORRECT_NONCE(94), - CHAT(999), + INVALID_TIMESTAMP_SIGNATURE(95), + ADDRESS_BLOCKED(96), + NAME_BLOCKED(97), + INVALID_BUT_OK(999), NOT_YET_RELEASED(1000); public final int value; @@ -311,6 +317,10 @@ public TransactionData getTransactionData() { return this.transactionData; } + public void setRepository(Repository repository) { + this.repository = repository; + } + // More information public static long getDeadline(TransactionData transactionData) { @@ -324,7 +334,7 @@ public long getDeadline() { /** Returns whether transaction's fee is at least minimum unit fee as specified in blockchain config. */ public boolean hasMinimumFee() { - return this.transactionData.getFee() >= BlockChain.getInstance().getUnitFee(); + return this.transactionData.getFee() >= this.getUnitFee(this.transactionData.getTimestamp()); } public long feePerByte() { @@ -337,9 +347,13 @@ public long feePerByte() { /** Returns whether transaction's fee is at least amount needed to cover byte-length of transaction. */ public boolean hasMinimumFeePerByte() { - long unitFee = BlockChain.getInstance().getUnitFee(); + long unitFee = this.getUnitFee(this.transactionData.getTimestamp()); int maxBytePerUnitFee = BlockChain.getInstance().getMaxBytesPerUnitFee(); + // If the unit fee is zero, any fee is enough to cover the byte-length of the transaction + if (unitFee == 0) { + return true; + } return this.feePerByte() >= maxBytePerUnitFee / unitFee; } @@ -355,7 +369,18 @@ public long calcRecommendedFee() { int unitFeeCount = ((dataLength - 1) / maxBytePerUnitFee) + 1; - return BlockChain.getInstance().getUnitFee() * unitFeeCount; + return this.getUnitFee(this.transactionData.getTimestamp()) * unitFeeCount; + } + + /** + * Calculate unit fee for a given transaction type + * + * FUTURE: add "accountLevel" parameter if needed - the level of the transaction creator + * @param timestamp - the transaction's timestamp (currently not used) + * @return + */ + public long getUnitFee(Long timestamp) { + return BlockChain.getInstance().getUnitFee(); } /** @@ -368,6 +393,9 @@ public long calcRecommendedFee() { * @return transaction version number */ public static int getVersionByTimestamp(long timestamp) { + if (timestamp >= BlockChain.getInstance().getTransactionV5Timestamp()) { + return 5; + } return 4; } @@ -602,7 +630,8 @@ private int countUnconfirmedByCreator(PublicKeyAccount creator) throws DataExcep public static List getUnconfirmedTransactions(Repository repository) throws DataException { BlockData latestBlockData = repository.getBlockRepository().getLastBlock(); - List unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(); + EnumSet excludedTxTypes = EnumSet.of(TransactionType.CHAT, TransactionType.PRESENCE); + List unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes); unconfirmedTransactions.sort(getDataComparator()); @@ -762,15 +791,20 @@ public Boolean getApprovalDecision() throws DataException { /** * Import into our repository as a new, unconfirmed transaction. *

- * Calls repository.saveChanges() + * @implSpec blocks to obtain blockchain lock + *

+ * If transaction is valid, then: + *

    + *
  • calls {@link Repository#discardChanges()}
    • + *
  • calls {@link Controller#onNewTransaction(TransactionData, Peer)}
    • + *
* * @throws DataException */ public ValidationResult importAsUnconfirmed() throws DataException { // Attempt to acquire blockchain lock ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - if (!blockchainLock.tryLock()) - return ValidationResult.NO_BLOCKCHAIN_LOCK; + blockchainLock.lock(); try { // Check transaction doesn't already exist @@ -780,6 +814,8 @@ public ValidationResult importAsUnconfirmed() throws DataException { // Fix up approval status this.setInitialApprovalStatus(); + this.preProcess(); + ValidationResult validationResult = this.isValidUnconfirmed(); if (validationResult != ValidationResult.OK) return validationResult; @@ -797,22 +833,47 @@ public ValidationResult importAsUnconfirmed() throws DataException { repository.getTransactionRepository().save(transactionData); repository.getTransactionRepository().unconfirmTransaction(transactionData); - /* - * If CHAT transaction then ensure there's at least a skeleton account so people - * can retrieve sender's public key using address, even if all their messages - * expire. - */ - if (transactionData.getType() == TransactionType.CHAT) - this.getCreator().ensureAccount(); + this.onImportAsUnconfirmed(); repository.saveChanges(); + // Notify controller of new transaction + Controller.getInstance().onNewTransaction(transactionData); + return ValidationResult.OK; } finally { + /* + * We call discardChanges() to restart repository 'transaction', discarding any + * transactional table locks, hence reducing possibility of deadlock or + * "serialization failure" with HSQLDB due to reads. + * + * "Serialization failure" most likely caused by existing transaction check above, + * where multiple threads are importing transactions + * and one thread finds existing an transaction, returns (unlocking blockchain lock), + * then another thread immediately obtains lock, tries to delete above existing transaction + * (e.g. older PRESENCE transaction) but can't because first thread's repository + * session still has row-lock on existing transaction and hasn't yet closed + * repository session. Deadlock caused by race condition. + * + * Hence we clear any repository-based locks before releasing blockchain lock. + */ + repository.discardChanges(); + blockchainLock.unlock(); } } + /** + * Callback for when a transaction is imported as unconfirmed. + *

+ * Called after transaction is added to repository, but before commit. + *

+ * Blockchain lock is being held during this time. + */ + protected void onImportAsUnconfirmed() throws DataException { + /* To be optionally overridden */ + } + /** * Returns whether transaction can be added to the blockchain. *

@@ -855,6 +916,14 @@ public ValidationResult isProcessable() throws DataException { return ValidationResult.OK; } + /** + * * Pre-process a transaction before validating or processing the block + * This allows for any database integrity checks prior to validation. + * + * @throws DataException + */ + public abstract void preProcess() throws DataException; + /** * Actually process a transaction, updating the blockchain. *

diff --git a/src/main/java/org/qortal/transaction/TransferAssetTransaction.java b/src/main/java/org/qortal/transaction/TransferAssetTransaction.java index a2855a357..79d485a57 100644 --- a/src/main/java/org/qortal/transaction/TransferAssetTransaction.java +++ b/src/main/java/org/qortal/transaction/TransferAssetTransaction.java @@ -61,6 +61,11 @@ public ValidationResult isProcessable() throws DataException { return new Payment(this.repository).isProcessable(this.transferAssetTransactionData.getSenderPublicKey(), getPaymentData(), this.transferAssetTransactionData.getFee()); } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Wrap asset transfer as a payment and delegate processing to Payment class. diff --git a/src/main/java/org/qortal/transaction/TransferPrivsTransaction.java b/src/main/java/org/qortal/transaction/TransferPrivsTransaction.java index d64e953ee..f77dac15b 100644 --- a/src/main/java/org/qortal/transaction/TransferPrivsTransaction.java +++ b/src/main/java/org/qortal/transaction/TransferPrivsTransaction.java @@ -68,6 +68,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { Account sender = this.getSender(); diff --git a/src/main/java/org/qortal/transaction/UpdateAssetTransaction.java b/src/main/java/org/qortal/transaction/UpdateAssetTransaction.java index 2a7af23c1..16e5641d9 100644 --- a/src/main/java/org/qortal/transaction/UpdateAssetTransaction.java +++ b/src/main/java/org/qortal/transaction/UpdateAssetTransaction.java @@ -90,6 +90,11 @@ public ValidationResult isProcessable() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Update Asset diff --git a/src/main/java/org/qortal/transaction/UpdateGroupTransaction.java b/src/main/java/org/qortal/transaction/UpdateGroupTransaction.java index 6751be337..9664ccbf2 100644 --- a/src/main/java/org/qortal/transaction/UpdateGroupTransaction.java +++ b/src/main/java/org/qortal/transaction/UpdateGroupTransaction.java @@ -109,6 +109,11 @@ public ValidationResult isProcessable() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { // Update Group diff --git a/src/main/java/org/qortal/transaction/UpdateNameTransaction.java b/src/main/java/org/qortal/transaction/UpdateNameTransaction.java index ebfde97c6..c9eedbae0 100644 --- a/src/main/java/org/qortal/transaction/UpdateNameTransaction.java +++ b/src/main/java/org/qortal/transaction/UpdateNameTransaction.java @@ -2,9 +2,11 @@ import java.util.Collections; import java.util.List; +import java.util.Objects; import org.qortal.account.Account; import org.qortal.asset.Asset; +import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.crypto.Crypto; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.TransactionData; @@ -124,6 +126,22 @@ public ValidationResult isProcessable() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData; + + // Rebuild this name in the Names table from the transaction history + // This is necessary because in some rare cases names can be missing from the Names table after registration + // but we have been unable to reproduce the issue and track down the root cause + NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck(); + namesDatabaseIntegrityCheck.rebuildName(updateNameTransactionData.getName(), this.repository); + + if (!Objects.equals(updateNameTransactionData.getName(), updateNameTransactionData.getNewName())) { + // Renaming - so make sure the new name is rebuilt too + namesDatabaseIntegrityCheck.rebuildName(updateNameTransactionData.getNewName(), this.repository); + } + } + @Override public void process() throws DataException { // Update Name diff --git a/src/main/java/org/qortal/transaction/VoteOnPollTransaction.java b/src/main/java/org/qortal/transaction/VoteOnPollTransaction.java index 35447aa6b..89eec1844 100644 --- a/src/main/java/org/qortal/transaction/VoteOnPollTransaction.java +++ b/src/main/java/org/qortal/transaction/VoteOnPollTransaction.java @@ -92,6 +92,11 @@ public ValidationResult isValid() throws DataException { return ValidationResult.OK; } + @Override + public void preProcess() throws DataException { + // Nothing to do + } + @Override public void process() throws DataException { String pollName = this.voteOnPollTransactionData.getPollName(); diff --git a/src/main/java/org/qortal/transform/Transformer.java b/src/main/java/org/qortal/transform/Transformer.java index 341d545ba..e78d3284d 100644 --- a/src/main/java/org/qortal/transform/Transformer.java +++ b/src/main/java/org/qortal/transform/Transformer.java @@ -20,5 +20,6 @@ public abstract class Transformer { public static final int MD5_LENGTH = 16; public static final int SHA256_LENGTH = 32; + public static final int AES256_LENGTH = 32; } diff --git a/src/main/java/org/qortal/transform/block/BlockTransformer.java b/src/main/java/org/qortal/transform/block/BlockTransformer.java index 4c9601185..cce3e7d70 100644 --- a/src/main/java/org/qortal/transform/block/BlockTransformer.java +++ b/src/main/java/org/qortal/transform/block/BlockTransformer.java @@ -7,7 +7,6 @@ import java.util.Arrays; import java.util.List; -import org.qortal.account.PublicKeyAccount; import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.data.at.ATStateData; @@ -23,7 +22,6 @@ import org.qortal.utils.Serialization; import org.qortal.utils.Triple; -import com.google.common.primitives.Bytes; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; @@ -76,19 +74,30 @@ public static Triple, List> fromBy } /** - * Extract block data and transaction data from serialized bytes. - * + * Extract block data and transaction data from serialized bytes containing a single block. + * * @param bytes * @return BlockData and a List of transactions. * @throws TransformationException */ public static Triple, List> fromByteBuffer(ByteBuffer byteBuffer) throws TransformationException { + return BlockTransformer.fromByteBuffer(byteBuffer, true); + } + + /** + * Extract block data and transaction data from serialized bytes containing one or more blocks. + * + * @param bytes + * @return the next block's BlockData and a List of transactions. + * @throws TransformationException + */ + public static Triple, List> fromByteBuffer(ByteBuffer byteBuffer, boolean finalBlockInBuffer) throws TransformationException { int version = byteBuffer.getInt(); - if (byteBuffer.remaining() < BASE_LENGTH + AT_BYTES_LENGTH - VERSION_LENGTH) + if (finalBlockInBuffer && byteBuffer.remaining() < BASE_LENGTH + AT_BYTES_LENGTH - VERSION_LENGTH) throw new TransformationException("Byte data too short for Block"); - if (byteBuffer.remaining() > BlockChain.getInstance().getMaxBlockSize()) + if (finalBlockInBuffer && byteBuffer.remaining() > BlockChain.getInstance().getMaxBlockSize()) throw new TransformationException("Byte data too long for Block"); long timestamp = byteBuffer.getLong(); @@ -212,7 +221,8 @@ public static Triple, List> fromBy byteBuffer.get(onlineAccountsSignatures); } - if (byteBuffer.hasRemaining()) + // We should only complain about excess byte data if we aren't expecting more blocks in this ByteBuffer + if (finalBlockInBuffer && byteBuffer.hasRemaining()) throw new TransformationException("Excess byte data found after parsing Block"); // We don't have a height! @@ -328,33 +338,38 @@ public static byte[] toBytes(Block block) throws TransformationException { } } - public static byte[] getMinterSignatureFromReference(byte[] blockReference) { - return Arrays.copyOf(blockReference, MINTER_SIGNATURE_LENGTH); + private static byte[] getReferenceBytesForMinterSignature(int blockHeight, byte[] reference) { + int newBlockSigTriggerHeight = BlockChain.getInstance().getNewBlockSigHeight(); + + return blockHeight >= newBlockSigTriggerHeight + // 'new' block sig uses all of previous block's signature + ? reference + // 'old' block sig only uses first 64 bytes of previous block's signature + : Arrays.copyOf(reference, MINTER_SIGNATURE_LENGTH); } - public static byte[] getBytesForMinterSignature(BlockData blockData) throws TransformationException { - byte[] minterSignature = getMinterSignatureFromReference(blockData.getReference()); - PublicKeyAccount minter = new PublicKeyAccount(null, blockData.getMinterPublicKey()); + public static byte[] getBytesForMinterSignature(BlockData blockData) { + byte[] referenceBytes = getReferenceBytesForMinterSignature(blockData.getHeight(), blockData.getReference()); - return getBytesForMinterSignature(minterSignature, minter, blockData.getEncodedOnlineAccounts()); + return getBytesForMinterSignature(referenceBytes, blockData.getMinterPublicKey(), blockData.getEncodedOnlineAccounts()); } - public static byte[] getBytesForMinterSignature(byte[] minterSignature, PublicKeyAccount minter, byte[] encodedOnlineAccounts) - throws TransformationException { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(MINTER_SIGNATURE_LENGTH + MINTER_PUBLIC_KEY_LENGTH + encodedOnlineAccounts.length); + public static byte[] getBytesForMinterSignature(BlockData parentBlockData, byte[] minterPublicKey, byte[] encodedOnlineAccounts) { + byte[] referenceBytes = getReferenceBytesForMinterSignature(parentBlockData.getHeight() + 1, parentBlockData.getSignature()); - bytes.write(minterSignature); + return getBytesForMinterSignature(referenceBytes, minterPublicKey, encodedOnlineAccounts); + } - // We're padding here just in case the minter is the genesis account whose public key is only 8 bytes long. - bytes.write(Bytes.ensureCapacity(minter.getPublicKey(), MINTER_PUBLIC_KEY_LENGTH, 0)); + private static byte[] getBytesForMinterSignature(byte[] referenceBytes, byte[] minterPublicKey, byte[] encodedOnlineAccounts) { + byte[] bytes = new byte[referenceBytes.length + MINTER_PUBLIC_KEY_LENGTH + encodedOnlineAccounts.length]; - bytes.write(encodedOnlineAccounts); + System.arraycopy(referenceBytes, 0, bytes, 0, referenceBytes.length); - return bytes.toByteArray(); - } catch (IOException e) { - throw new TransformationException(e); - } + System.arraycopy(minterPublicKey, 0, bytes, referenceBytes.length, MINTER_PUBLIC_KEY_LENGTH); + + System.arraycopy(encodedOnlineAccounts, 0, bytes, referenceBytes.length + MINTER_PUBLIC_KEY_LENGTH, encodedOnlineAccounts.length); + + return bytes; } public static byte[] getBytesForTransactionsSignature(Block block) throws TransformationException { diff --git a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java index 3402ca66d..c3190f03e 100644 --- a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java +++ b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java @@ -6,12 +6,15 @@ import java.util.ArrayList; import java.util.List; +import com.google.common.base.Utf8; +import org.qortal.arbitrary.misc.Service; import org.qortal.crypto.Crypto; import org.qortal.data.PaymentData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.DataType; +import org.qortal.naming.Name; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; @@ -26,12 +29,23 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { // Property lengths private static final int SERVICE_LENGTH = INT_LENGTH; + private static final int NONCE_LENGTH = INT_LENGTH; private static final int DATA_TYPE_LENGTH = BYTE_LENGTH; private static final int DATA_SIZE_LENGTH = INT_LENGTH; + private static final int RAW_DATA_SIZE_LENGTH = INT_LENGTH; + private static final int METADATA_HASH_SIZE_LENGTH = INT_LENGTH; private static final int NUMBER_PAYMENTS_LENGTH = INT_LENGTH; + private static final int NAME_SIZE_LENGTH = INT_LENGTH; + private static final int IDENTIFIER_SIZE_LENGTH = INT_LENGTH; + private static final int COMPRESSION_LENGTH = INT_LENGTH; + private static final int METHOD_LENGTH = INT_LENGTH; + private static final int SECRET_LENGTH = INT_LENGTH; // TODO: wtf? private static final int EXTRAS_LENGTH = SERVICE_LENGTH + DATA_TYPE_LENGTH + DATA_SIZE_LENGTH; + private static final int EXTRAS_V5_LENGTH = NONCE_LENGTH + NAME_SIZE_LENGTH + IDENTIFIER_SIZE_LENGTH + + METHOD_LENGTH + SECRET_LENGTH + COMPRESSION_LENGTH + RAW_DATA_SIZE_LENGTH + METADATA_HASH_SIZE_LENGTH; + protected static final TransactionLayout layout; static { @@ -41,8 +55,18 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { layout.add("transaction's groupID", TransformationType.INT); layout.add("reference", TransformationType.SIGNATURE); layout.add("sender's public key", TransformationType.PUBLIC_KEY); - layout.add("number of payments", TransformationType.INT); + layout.add("nonce", TransformationType.INT); // Version 5+ + layout.add("name length", TransformationType.INT); // Version 5+ + layout.add("name", TransformationType.DATA); // Version 5+ + layout.add("identifier length", TransformationType.INT); // Version 5+ + layout.add("identifier", TransformationType.DATA); // Version 5+ + layout.add("method", TransformationType.INT); // Version 5+ + layout.add("secret length", TransformationType.INT); // Version 5+ + layout.add("secret", TransformationType.DATA); // Version 5+ + layout.add("compression", TransformationType.INT); // Version 5+ + + layout.add("number of payments", TransformationType.INT); layout.add("* recipient", TransformationType.ADDRESS); layout.add("* asset ID of payment", TransformationType.LONG); layout.add("* payment amount", TransformationType.AMOUNT); @@ -51,6 +75,11 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { layout.add("is data raw?", TransformationType.BOOLEAN); layout.add("data length", TransformationType.INT); layout.add("data", TransformationType.DATA); + + layout.add("raw data size", TransformationType.INT); // Version 5+ + layout.add("metadata hash length", TransformationType.INT); // Version 5+ + layout.add("metadata hash", TransformationType.DATA); // Version 5+ + layout.add("fee", TransformationType.AMOUNT); layout.add("signature", TransformationType.SIGNATURE); } @@ -67,6 +96,32 @@ public static TransactionData fromByteBuffer(ByteBuffer byteBuffer) throws Trans byte[] senderPublicKey = Serialization.deserializePublicKey(byteBuffer); + int nonce = 0; + String name = null; + String identifier = null; + ArbitraryTransactionData.Method method = null; + byte[] secret = null; + ArbitraryTransactionData.Compression compression = null; + + if (version >= 5) { + nonce = byteBuffer.getInt(); + + name = Serialization.deserializeSizedStringV2(byteBuffer, Name.MAX_NAME_SIZE); + + identifier = Serialization.deserializeSizedStringV2(byteBuffer, ArbitraryTransaction.MAX_IDENTIFIER_LENGTH); + + method = ArbitraryTransactionData.Method.valueOf(byteBuffer.getInt()); + + int secretLength = byteBuffer.getInt(); + + if (secretLength > 0) { + secret = new byte[secretLength]; + byteBuffer.get(secret); + } + + compression = ArbitraryTransactionData.Compression.valueOf(byteBuffer.getInt()); + } + // Always return a list of payments, even if empty List payments = new ArrayList<>(); if (version != 1) { @@ -76,7 +131,7 @@ public static TransactionData fromByteBuffer(ByteBuffer byteBuffer) throws Trans payments.add(PaymentTransformer.fromByteBuffer(byteBuffer)); } - int service = byteBuffer.getInt(); + Service service = Service.valueOf(byteBuffer.getInt()); // We might be receiving hash of data instead of actual raw data boolean isRaw = byteBuffer.get() != 0; @@ -91,6 +146,20 @@ public static TransactionData fromByteBuffer(ByteBuffer byteBuffer) throws Trans byte[] data = new byte[dataSize]; byteBuffer.get(data); + int size = 0; + byte[] metadataHash = null; + + if (version >= 5) { + size = byteBuffer.getInt(); + + int metadataHashLength = byteBuffer.getInt(); + + if (metadataHashLength > 0) { + metadataHash = new byte[metadataHashLength]; + byteBuffer.get(metadataHash); + } + } + long fee = byteBuffer.getLong(); byte[] signature = new byte[SIGNATURE_LENGTH]; @@ -98,13 +167,24 @@ public static TransactionData fromByteBuffer(ByteBuffer byteBuffer) throws Trans BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, senderPublicKey, fee, signature); - return new ArbitraryTransactionData(baseTransactionData, version, service, data, dataType, payments); + return new ArbitraryTransactionData(baseTransactionData, version, service, nonce, size, name, identifier, + method, secret, compression, data, dataType, metadataHash, payments); } public static int getDataLength(TransactionData transactionData) throws TransformationException { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - int length = getBaseLength(transactionData) + EXTRAS_LENGTH + arbitraryTransactionData.getData().length; + int nameLength = (arbitraryTransactionData.getName() != null) ? Utf8.encodedLength(arbitraryTransactionData.getName()) : 0; + int identifierLength = (arbitraryTransactionData.getIdentifier() != null) ? Utf8.encodedLength(arbitraryTransactionData.getIdentifier()) : 0; + int secretLength = (arbitraryTransactionData.getSecret() != null) ? arbitraryTransactionData.getSecret().length : 0; + int dataLength = (arbitraryTransactionData.getData() != null) ? arbitraryTransactionData.getData().length : 0; + int metadataHashLength = (arbitraryTransactionData.getMetadataHash() != null) ? arbitraryTransactionData.getMetadataHash().length : 0; + + int length = getBaseLength(transactionData) + EXTRAS_LENGTH + nameLength + identifierLength + secretLength + dataLength + metadataHashLength; + + if (arbitraryTransactionData.getVersion() >= 5) { + length += EXTRAS_V5_LENGTH; + } // Optional payments length += NUMBER_PAYMENTS_LENGTH + arbitraryTransactionData.getPayments().size() * PaymentTransformer.getDataLength(); @@ -120,19 +200,51 @@ public static byte[] toBytes(TransactionData transactionData) throws Transformat transformCommonBytes(transactionData, bytes); + if (arbitraryTransactionData.getVersion() >= 5) { + bytes.write(Ints.toByteArray(arbitraryTransactionData.getNonce())); + + Serialization.serializeSizedStringV2(bytes, arbitraryTransactionData.getName()); + + Serialization.serializeSizedStringV2(bytes, arbitraryTransactionData.getIdentifier()); + + bytes.write(Ints.toByteArray(arbitraryTransactionData.getMethod().value)); + + byte[] secret = arbitraryTransactionData.getSecret(); + int secretLength = (secret != null) ? secret.length : 0; + bytes.write(Ints.toByteArray(secretLength)); + + if (secretLength > 0) { + bytes.write(secret); + } + + bytes.write(Ints.toByteArray(arbitraryTransactionData.getCompression().value)); + } + List payments = arbitraryTransactionData.getPayments(); bytes.write(Ints.toByteArray(payments.size())); for (PaymentData paymentData : payments) bytes.write(PaymentTransformer.toBytes(paymentData)); - bytes.write(Ints.toByteArray(arbitraryTransactionData.getService())); + bytes.write(Ints.toByteArray(arbitraryTransactionData.getService().value)); bytes.write((byte) (arbitraryTransactionData.getDataType() == DataType.RAW_DATA ? 1 : 0)); bytes.write(Ints.toByteArray(arbitraryTransactionData.getData().length)); bytes.write(arbitraryTransactionData.getData()); + if (arbitraryTransactionData.getVersion() >= 5) { + bytes.write(Ints.toByteArray(arbitraryTransactionData.getSize())); + + byte[] metadataHash = arbitraryTransactionData.getMetadataHash(); + int metadataHashLength = (metadataHash != null) ? metadataHash.length : 0; + bytes.write(Ints.toByteArray(metadataHashLength)); + + if (metadataHashLength > 0) { + bytes.write(metadataHash); + } + } + bytes.write(Longs.toByteArray(arbitraryTransactionData.getFee())); if (arbitraryTransactionData.getSignature() != null) @@ -159,6 +271,26 @@ protected static byte[] toBytesForSigningImpl(TransactionData transactionData) t transformCommonBytes(arbitraryTransactionData, bytes); + if (arbitraryTransactionData.getVersion() >= 5) { + bytes.write(Ints.toByteArray(arbitraryTransactionData.getNonce())); + + Serialization.serializeSizedStringV2(bytes, arbitraryTransactionData.getName()); + + Serialization.serializeSizedStringV2(bytes, arbitraryTransactionData.getIdentifier()); + + bytes.write(Ints.toByteArray(arbitraryTransactionData.getMethod().value)); + + byte[] secret = arbitraryTransactionData.getSecret(); + int secretLength = (secret != null) ? secret.length : 0; + bytes.write(Ints.toByteArray(secretLength)); + + if (secretLength > 0) { + bytes.write(secret); + } + + bytes.write(Ints.toByteArray(arbitraryTransactionData.getCompression().value)); + } + if (arbitraryTransactionData.getVersion() != 1) { List payments = arbitraryTransactionData.getPayments(); bytes.write(Ints.toByteArray(payments.size())); @@ -167,7 +299,7 @@ protected static byte[] toBytesForSigningImpl(TransactionData transactionData) t bytes.write(PaymentTransformer.toBytes(paymentData)); } - bytes.write(Ints.toByteArray(arbitraryTransactionData.getService())); + bytes.write(Ints.toByteArray(arbitraryTransactionData.getService().value)); bytes.write(Ints.toByteArray(arbitraryTransactionData.getData().length)); @@ -182,6 +314,18 @@ protected static byte[] toBytesForSigningImpl(TransactionData transactionData) t break; } + if (arbitraryTransactionData.getVersion() >= 5) { + bytes.write(Ints.toByteArray(arbitraryTransactionData.getSize())); + + byte[] metadataHash = arbitraryTransactionData.getMetadataHash(); + int metadataHashLength = (metadataHash != null) ? metadataHash.length : 0; + bytes.write(Ints.toByteArray(metadataHashLength)); + + if (metadataHashLength > 0) { + bytes.write(metadataHash); + } + } + bytes.write(Longs.toByteArray(arbitraryTransactionData.getFee())); // Never append signature @@ -192,4 +336,13 @@ protected static byte[] toBytesForSigningImpl(TransactionData transactionData) t } } + public static void clearNonce(byte[] transactionBytes) { + int nonceIndex = TYPE_LENGTH + TIMESTAMP_LENGTH + GROUPID_LENGTH + REFERENCE_LENGTH + PUBLIC_KEY_LENGTH; + + transactionBytes[nonceIndex++] = (byte) 0; + transactionBytes[nonceIndex++] = (byte) 0; + transactionBytes[nonceIndex++] = (byte) 0; + transactionBytes[nonceIndex++] = (byte) 0; + } + } diff --git a/src/main/java/org/qortal/transform/transaction/PresenceTransactionTransformer.java b/src/main/java/org/qortal/transform/transaction/PresenceTransactionTransformer.java new file mode 100644 index 000000000..bf69d1025 --- /dev/null +++ b/src/main/java/org/qortal/transform/transaction/PresenceTransactionTransformer.java @@ -0,0 +1,108 @@ +package org.qortal.transform.transaction; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.transaction.PresenceTransaction.PresenceType; +import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.transform.TransformationException; +import org.qortal.utils.Serialization; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; + +public class PresenceTransactionTransformer extends TransactionTransformer { + + // Property lengths + private static final int NONCE_LENGTH = INT_LENGTH; + private static final int PRESENCE_TYPE_LENGTH = BYTE_LENGTH; + private static final int TIMESTAMP_SIGNATURE_LENGTH = SIGNATURE_LENGTH; + + private static final int EXTRAS_LENGTH = NONCE_LENGTH + PRESENCE_TYPE_LENGTH + TIMESTAMP_SIGNATURE_LENGTH; + + protected static final TransactionLayout layout; + + static { + layout = new TransactionLayout(); + layout.add("txType: " + TransactionType.PRESENCE.valueString, TransformationType.INT); + layout.add("timestamp", TransformationType.TIMESTAMP); + layout.add("transaction's groupID", TransformationType.INT); + layout.add("reference", TransformationType.SIGNATURE); + layout.add("sender's public key", TransformationType.PUBLIC_KEY); + layout.add("proof-of-work nonce", TransformationType.INT); + layout.add("presence type (reward-share=0, trade-bot=1)", TransformationType.BYTE); + layout.add("timestamp-signature", TransformationType.SIGNATURE); + layout.add("fee", TransformationType.AMOUNT); + layout.add("signature", TransformationType.SIGNATURE); + } + + public static TransactionData fromByteBuffer(ByteBuffer byteBuffer) throws TransformationException { + long timestamp = byteBuffer.getLong(); + + int txGroupId = byteBuffer.getInt(); + + byte[] reference = new byte[REFERENCE_LENGTH]; + byteBuffer.get(reference); + + byte[] senderPublicKey = Serialization.deserializePublicKey(byteBuffer); + + int nonce = byteBuffer.getInt(); + + PresenceType presenceType = PresenceType.valueOf(byteBuffer.get()); + + byte[] timestampSignature = new byte[SIGNATURE_LENGTH]; + byteBuffer.get(timestampSignature); + + long fee = byteBuffer.getLong(); + + byte[] signature = new byte[SIGNATURE_LENGTH]; + byteBuffer.get(signature); + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, senderPublicKey, fee, signature); + + return new PresenceTransactionData(baseTransactionData, nonce, presenceType, timestampSignature); + } + + public static int getDataLength(TransactionData transactionData) { + return getBaseLength(transactionData) + EXTRAS_LENGTH; + } + + public static byte[] toBytes(TransactionData transactionData) throws TransformationException { + try { + PresenceTransactionData presenceTransactionData = (PresenceTransactionData) transactionData; + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + transformCommonBytes(transactionData, bytes); + + bytes.write(Ints.toByteArray(presenceTransactionData.getNonce())); + + bytes.write(presenceTransactionData.getPresenceType().value); + + bytes.write(presenceTransactionData.getTimestampSignature()); + + bytes.write(Longs.toByteArray(presenceTransactionData.getFee())); + + if (presenceTransactionData.getSignature() != null) + bytes.write(presenceTransactionData.getSignature()); + + return bytes.toByteArray(); + } catch (IOException | ClassCastException e) { + throw new TransformationException(e); + } + } + + public static void clearNonce(byte[] transactionBytes) { + int nonceIndex = TYPE_LENGTH + TIMESTAMP_LENGTH + GROUPID_LENGTH + REFERENCE_LENGTH + PUBLIC_KEY_LENGTH; + + transactionBytes[nonceIndex++] = (byte) 0; + transactionBytes[nonceIndex++] = (byte) 0; + transactionBytes[nonceIndex++] = (byte) 0; + transactionBytes[nonceIndex++] = (byte) 0; + } + +} diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java new file mode 100644 index 000000000..9b81bd683 --- /dev/null +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -0,0 +1,413 @@ +package org.qortal.utils; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataFileChunk; +import org.qortal.arbitrary.misc.Service; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.settings.Settings; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + + +public class ArbitraryTransactionUtils { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryTransactionUtils.class); + + public static ArbitraryTransactionData fetchTransactionData(final Repository repository, final byte[] signature) { + try { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (!(transactionData instanceof ArbitraryTransactionData)) + return null; + + return (ArbitraryTransactionData) transactionData; + + } catch (DataException e) { + LOGGER.error("Repository issue when fetching arbitrary transaction data", e); + return null; + } + } + + public static ArbitraryTransactionData fetchLatestPut(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { + if (arbitraryTransactionData == null) { + return null; + } + + String name = arbitraryTransactionData.getName(); + Service service = arbitraryTransactionData.getService(); + String identifier = arbitraryTransactionData.getIdentifier(); + + if (name == null || service == null) { + return null; + } + + // Get the most recent PUT for this name and service + ArbitraryTransactionData latestPut; + try { + latestPut = repository.getArbitraryRepository() + .getLatestTransaction(name, service, ArbitraryTransactionData.Method.PUT, identifier); + } catch (DataException e) { + return null; + } + + return latestPut; + } + + public static boolean hasMoreRecentPutTransaction(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { + byte[] signature = arbitraryTransactionData.getSignature(); + if (signature == null) { + // We can't make a sensible decision without a signature + // so it's best to assume there is nothing newer + return false; + } + + ArbitraryTransactionData latestPut = ArbitraryTransactionUtils.fetchLatestPut(repository, arbitraryTransactionData); + if (latestPut == null) { + return false; + } + + // If the latest PUT transaction has a newer timestamp, it will override the existing transaction + // Any data relating to the older transaction is no longer needed + boolean hasNewerPut = (latestPut.getTimestamp() > arbitraryTransactionData.getTimestamp()); + return hasNewerPut; + } + + public static boolean completeFileExists(ArbitraryTransactionData transactionData) throws DataException { + if (transactionData == null) { + return false; + } + + byte[] digest = transactionData.getData(); + byte[] signature = transactionData.getSignature(); + + // Load complete file + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + return arbitraryDataFile.exists(); + + } + + public static boolean allChunksExist(ArbitraryTransactionData transactionData) throws DataException { + if (transactionData == null) { + return false; + } + + byte[] digest = transactionData.getData(); + byte[] metadataHash = transactionData.getMetadataHash(); + byte[] signature = transactionData.getSignature(); + + // Load complete file and chunks + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + + return arbitraryDataFile.allChunksExist(); + } + + public static boolean anyChunksExist(ArbitraryTransactionData transactionData) throws DataException { + if (transactionData == null) { + return false; + } + + byte[] digest = transactionData.getData(); + byte[] metadataHash = transactionData.getMetadataHash(); + byte[] signature = transactionData.getSignature(); + + if (metadataHash == null) { + // This file doesn't have any metadata/chunks, which means none exist + return false; + } + + // Load complete file and chunks + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + + return arbitraryDataFile.anyChunksExist(); + } + + public static int ourChunkCount(ArbitraryTransactionData transactionData) throws DataException { + if (transactionData == null) { + return 0; + } + + byte[] digest = transactionData.getData(); + byte[] metadataHash = transactionData.getMetadataHash(); + byte[] signature = transactionData.getSignature(); + + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + + // Find the folder containing the files + Path parentPath = arbitraryDataFile.getFilePath().getParent(); + String[] files = parentPath.toFile().list(); + if (files == null) { + return 0; + } + + // Remove the original copy indicator file if it exists + files = ArrayUtils.removeElement(files, ".original"); + + int count = files.length; + + // If the complete file exists (and this transaction has chunks), subtract it from the count + if (arbitraryDataFile.chunkCount() > 0 && arbitraryDataFile.exists()) { + // We are only measuring the individual chunks, not the joined file + count -= 1; + } + + return count; + } + + public static int totalChunkCount(ArbitraryTransactionData transactionData) throws DataException { + if (transactionData == null) { + return 0; + } + + byte[] digest = transactionData.getData(); + byte[] metadataHash = transactionData.getMetadataHash(); + byte[] signature = transactionData.getSignature(); + + if (metadataHash == null) { + // This file doesn't have any metadata, therefore it has a single (complete) chunk + return 1; + } + + // Load complete file and chunks + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + + return arbitraryDataFile.chunkCount() + 1; // +1 for the metadata file + } + + public static boolean isFileRecent(Path filePath, long now, long cleanupAfter) { + try { + BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); + long timeSinceCreated = now - attr.creationTime().toMillis(); + long timeSinceModified = now - attr.lastModifiedTime().toMillis(); + //LOGGER.info(String.format("timeSinceCreated for path %s is %d. cleanupAfter: %d", filePath, timeSinceCreated, cleanupAfter)); + + // Check if the file has been created or modified recently + if (timeSinceCreated > cleanupAfter) { + return false; + } + if (timeSinceModified > cleanupAfter) { + return false; + } + + } catch (IOException e) { + // Can't read file attributes, so assume it's recent so that we don't delete something accidentally + } + return true; + } + + public static boolean isFileHashRecent(byte[] hash, byte[] signature, long now, long cleanupAfter) throws DataException { + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); + if (arbitraryDataFile == null || !arbitraryDataFile.exists()) { + // No hash, or file doesn't exist, so it's not recent + return false; + } + + Path filePath = arbitraryDataFile.getFilePath(); + return ArbitraryTransactionUtils.isFileRecent(filePath, now, cleanupAfter); + } + + public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException { + byte[] completeHash = arbitraryTransactionData.getData(); + byte[] signature = arbitraryTransactionData.getSignature(); + + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature); + + if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, signature, now, cleanupAfter)) { + LOGGER.info("Deleting file {} because it can be rebuilt from chunks " + + "if needed", Base58.encode(completeHash)); + + arbitraryDataFile.delete(); + } + } + + public static void deleteCompleteFileAndChunks(ArbitraryTransactionData arbitraryTransactionData) throws DataException { + byte[] completeHash = arbitraryTransactionData.getData(); + byte[] metadataHash = arbitraryTransactionData.getMetadataHash(); + byte[] signature = arbitraryTransactionData.getSignature(); + + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + arbitraryDataFile.deleteAll(); + } + + public static void convertFileToChunks(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException { + byte[] completeHash = arbitraryTransactionData.getData(); + byte[] metadataHash = arbitraryTransactionData.getMetadataHash(); + byte[] signature = arbitraryTransactionData.getSignature(); + + // Find the expected chunk hashes + ArbitraryDataFile expectedDataFile = ArbitraryDataFile.fromHash(completeHash, signature); + expectedDataFile.setMetadataHash(metadataHash); + + if (metadataHash == null || !expectedDataFile.getMetadataFile().exists()) { + // We don't have the metadata file, or this transaction doesn't have one - nothing to do + return; + } + + // Split the file into chunks + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature); + int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE); + if (chunkCount > 1) { + LOGGER.info(String.format("Successfully split %s into %d chunk%s", + Base58.encode(completeHash), chunkCount, (chunkCount == 1 ? "" : "s"))); + + // Verify that the chunk hashes match those in the transaction + byte[] chunkHashes = expectedDataFile.chunkHashes(); + if (chunkHashes != null && Arrays.equals(chunkHashes, arbitraryDataFile.chunkHashes())) { + // Ensure they exist on disk + if (arbitraryDataFile.allChunksExist()) { + + // Now delete the original file if it's not recent + if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, signature, now, cleanupAfter)) { + LOGGER.info("Deleting file {} because it can now be rebuilt from " + + "chunks if needed", Base58.encode(completeHash)); + + ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, cleanupAfter); + } + else { + // File might be in use. It's best to leave it and it it will be cleaned up later. + } + } + } + } + } + + /** + * When first uploaded, files go into a _misc folder as they are not yet associated with a + * transaction signature. Once the transaction is broadcast, they need to be moved to the + * correct location, keyed by the transaction signature. + * @param arbitraryTransactionData + * @return + * @throws DataException + */ + public static int checkAndRelocateMiscFiles(ArbitraryTransactionData arbitraryTransactionData) { + int filesRelocatedCount = 0; + + try { + // Load hashes + byte[] digest = arbitraryTransactionData.getData(); + byte[] metadataHash = arbitraryTransactionData.getMetadataHash(); + + // Load signature + byte[] signature = arbitraryTransactionData.getSignature(); + + // Check if any files for this transaction exist in the misc folder + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, null); + arbitraryDataFile.setMetadataHash(metadataHash); + + if (arbitraryDataFile.anyChunksExist()) { + // At least one chunk exists in the misc folder - move them + for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) { + if (chunk.exists()) { + // Determine the correct path by initializing a new ArbitraryDataFile instance with the signature + ArbitraryDataFile newChunk = ArbitraryDataFile.fromHash(chunk.getHash(), signature); + Path oldPath = chunk.getFilePath(); + Path newPath = newChunk.getFilePath(); + + // Ensure parent directories exist, then copy the file + LOGGER.info("Relocating chunk from {} to {}...", oldPath, newPath); + Files.createDirectories(newPath.getParent()); + Files.move(oldPath, newPath, REPLACE_EXISTING); + filesRelocatedCount++; + + // Delete empty parent directories + FilesystemUtils.safeDeleteEmptyParentDirectories(oldPath); + } + } + } + // Also move the complete file if it exists + if (arbitraryDataFile.exists()) { + // Determine the correct path by initializing a new ArbitraryDataFile instance with the signature + ArbitraryDataFile newCompleteFile = ArbitraryDataFile.fromHash(arbitraryDataFile.getHash(), signature); + Path oldPath = arbitraryDataFile.getFilePath(); + Path newPath = newCompleteFile.getFilePath(); + + // Ensure parent directories exist, then copy the file + LOGGER.info("Relocating complete file from {} to {}...", oldPath, newPath); + Files.createDirectories(newPath.getParent()); + Files.move(oldPath, newPath, REPLACE_EXISTING); + filesRelocatedCount++; + + // Delete empty parent directories + FilesystemUtils.safeDeleteEmptyParentDirectories(oldPath); + } + + // Also move the metadata file if it exists + if (arbitraryDataFile.getMetadataFile() != null && arbitraryDataFile.getMetadataFile().exists()) { + // Determine the correct path by initializing a new ArbitraryDataFile instance with the signature + ArbitraryDataFile newCompleteFile = ArbitraryDataFile.fromHash(arbitraryDataFile.getMetadataHash(), signature); + Path oldPath = arbitraryDataFile.getMetadataFile().getFilePath(); + Path newPath = newCompleteFile.getFilePath(); + + // Ensure parent directories exist, then copy the file + LOGGER.info("Relocating metadata file from {} to {}...", oldPath, newPath); + Files.createDirectories(newPath.getParent()); + Files.move(oldPath, newPath, REPLACE_EXISTING); + filesRelocatedCount++; + + // Delete empty parent directories + FilesystemUtils.safeDeleteEmptyParentDirectories(oldPath); + } + + // If at least one file was relocated, we can assume that the data from this transaction + // originated from this node + if (filesRelocatedCount > 0) { + if (Settings.getInstance().isOriginalCopyIndicatorFileEnabled()) { + // Create a file in the same directory, to indicate that this is the original copy + LOGGER.info("Creating original copy indicator file..."); + ArbitraryDataFile completeFile = ArbitraryDataFile.fromHash(arbitraryDataFile.getHash(), signature); + Path parentDirectory = completeFile.getFilePath().getParent(); + File file = Paths.get(parentDirectory.toString(), ".original").toFile(); + file.createNewFile(); + } + } + } + catch (DataException | IOException e) { + LOGGER.info("Unable to check and relocate all files for signature {}: {}", + Base58.encode(arbitraryTransactionData.getSignature()), e.getMessage()); + } + + return filesRelocatedCount; + } + + public static List limitOffsetTransactions(List transactions, + Integer limit, Integer offset) { + if (limit != null && limit == 0) { + limit = null; + } + if (limit == null && offset == null) { + return transactions; + } + if (offset == null) { + offset = 0; + } + if (offset > transactions.size() - 1) { + return new ArrayList<>(); + } + + if (limit == null) { + return transactions.stream().skip(offset).collect(Collectors.toList()); + } + return transactions.stream().skip(offset).limit(limit).collect(Collectors.toList()); + } + +} diff --git a/src/main/java/org/qortal/utils/BIP39.java b/src/main/java/org/qortal/utils/BIP39.java deleted file mode 100644 index 488396eb6..000000000 --- a/src/main/java/org/qortal/utils/BIP39.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.qortal.utils; - -import java.util.ArrayList; -import java.util.List; - -import org.qortal.globalization.BIP39WordList; - -public class BIP39 { - - private static final int BITS_PER_WORD = 11; - - /** Convert BIP39 mnemonic to binary 'entropy' */ - public static byte[] decode(String[] phraseWords, String lang) { - if (lang == null) - lang = "en"; - - List wordList = BIP39WordList.INSTANCE.getByLang(lang); - if (wordList == null) - throw new IllegalStateException("BIP39 word list for lang '" + lang + "' unavailable"); - - byte[] entropy = new byte[(phraseWords.length * BITS_PER_WORD + 7) / 8]; - int byteIndex = 0; - int bitShift = 3; - - for (int i = 0; i < phraseWords.length; ++i) { - int wordListIndex = wordList.indexOf(phraseWords[i]); - if (wordListIndex == -1) - // Word not found - return null; - - entropy[byteIndex++] |= (byte) (wordListIndex >> bitShift); - - bitShift = 8 - bitShift; - if (bitShift >= 0) { - // Leftover fits inside one byte - entropy[byteIndex] |= (byte) ((wordListIndex << bitShift)); - bitShift = BITS_PER_WORD - bitShift; - } else { - // Leftover spread over next two bytes - bitShift = - bitShift; - entropy[byteIndex++] |= (byte) (wordListIndex >> bitShift); - - entropy[byteIndex] |= (byte) (wordListIndex << (8 - bitShift)); - bitShift = bitShift + BITS_PER_WORD - 8; - } - } - - return entropy; - } - - /** Convert binary entropy to BIP39 mnemonic */ - public static String encode(byte[] entropy, String lang) { - if (lang == null) - lang = "en"; - - List wordList = BIP39WordList.INSTANCE.getByLang(lang); - if (wordList == null) - throw new IllegalStateException("BIP39 word list for lang '" + lang + "' unavailable"); - - List phraseWords = new ArrayList<>(); - - int bitMask = 128; // MSB first - int byteIndex = 0; - while (true) { - int wordListIndex = 0; - for (int bitCount = 0; bitCount < BITS_PER_WORD; ++bitCount) { - wordListIndex <<= 1; - - if ((entropy[byteIndex] & bitMask) != 0) - ++wordListIndex; - - bitMask >>= 1; - if (bitMask == 0) { - bitMask = 128; - ++byteIndex; - - if (byteIndex >= entropy.length) - return String.join(" ", phraseWords); - } - } - - phraseWords.add(wordList.get(wordListIndex)); - } - } - -} diff --git a/src/main/java/org/qortal/utils/BitTwiddling.java b/src/main/java/org/qortal/utils/BitTwiddling.java index f13300c51..eda5b4f6a 100644 --- a/src/main/java/org/qortal/utils/BitTwiddling.java +++ b/src/main/java/org/qortal/utils/BitTwiddling.java @@ -26,9 +26,26 @@ public static byte[] toLEByteArray(int value) { return new byte[] { (byte) (value), (byte) (value >> 8), (byte) (value >> 16), (byte) (value >> 24) }; } + /** Convert int to big-endian byte array */ + public static byte[] toBEByteArray(int value) { + return new byte[] { (byte) (value >> 24), (byte) (value >> 16), (byte) (value >> 8), (byte) (value) }; + } + + /** Convert long to big-endian byte array */ + public static byte[] toBEByteArray(long value) { + return new byte[] { (byte) (value >> 56), (byte) (value >> 48), (byte) (value >> 40), (byte) (value >> 32), + (byte) (value >> 24), (byte) (value >> 16), (byte) (value >> 8), (byte) (value) }; + } + /** Convert little-endian bytes to int */ - public static int fromLEBytes(byte[] bytes, int offset) { + public static int intFromLEBytes(byte[] bytes, int offset) { return (bytes[offset] & 0xff) | (bytes[offset + 1] & 0xff) << 8 | (bytes[offset + 2] & 0xff) << 16 | (bytes[offset + 3] & 0xff) << 24; } + /** Convert big-endian bytes to long */ + public static long longFromBEBytes(byte[] bytes, int start) { + return (bytes[start] & 0xffL) << 56 | (bytes[start + 1] & 0xffL) << 48 | (bytes[start + 2] & 0xffL) << 40 | (bytes[start + 3] & 0xffL) << 32 + | (bytes[start + 4] & 0xffL) << 24 | (bytes[start + 5] & 0xffL) << 16 | (bytes[start + 6] & 0xffL) << 8 | (bytes[start + 7] & 0xffL); + } + } diff --git a/src/main/java/org/qortal/utils/BlockArchiveUtils.java b/src/main/java/org/qortal/utils/BlockArchiveUtils.java new file mode 100644 index 000000000..0beff026e --- /dev/null +++ b/src/main/java/org/qortal/utils/BlockArchiveUtils.java @@ -0,0 +1,78 @@ +package org.qortal.utils; + +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.BlockArchiveReader; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; + +import java.util.List; + +public class BlockArchiveUtils { + + /** + * importFromArchive + *

+ * Reads the requested block range from the archive + * and imports the BlockData and AT state data hashes + * This can be used to convert a block archive back + * into the HSQLDB, in order to make it SQL-compatible + * again. + *

+ * Note: calls discardChanges() and saveChanges(), so + * make sure that you commit any existing repository + * changes before calling this method. + * + * @param startHeight The earliest block to import + * @param endHeight The latest block to import + * @param repository A clean repository session + * @throws DataException + */ + public static void importFromArchive(int startHeight, int endHeight, Repository repository) throws DataException { + repository.discardChanges(); + final int requestedRange = endHeight+1-startHeight; + + List, List>> blockInfoList = + BlockArchiveReader.getInstance().fetchBlocksFromRange(startHeight, endHeight); + + // Ensure that we have received all of the requested blocks + if (blockInfoList == null || blockInfoList.isEmpty()) { + throw new IllegalStateException("No blocks found when importing from archive"); + } + if (blockInfoList.size() != requestedRange) { + throw new IllegalStateException("Non matching block count when importing from archive"); + } + Triple, List> firstBlock = blockInfoList.get(0); + if (firstBlock == null || firstBlock.getA().getHeight() != startHeight) { + throw new IllegalStateException("Non matching first block when importing from archive"); + } + if (blockInfoList.size() > 0) { + Triple, List> lastBlock = + blockInfoList.get(blockInfoList.size() - 1); + if (lastBlock == null || lastBlock.getA().getHeight() != endHeight) { + throw new IllegalStateException("Non matching last block when importing from archive"); + } + } + + // Everything seems okay, so go ahead with the import + for (Triple, List> blockInfo : blockInfoList) { + try { + // Save block + repository.getBlockRepository().save(blockInfo.getA()); + + // Save AT state data hashes + for (ATStateData atStateData : blockInfo.getC()) { + atStateData.setHeight(blockInfo.getA().getHeight()); + repository.getATRepository().save(atStateData); + } + + } catch (DataException e) { + repository.discardChanges(); + throw new IllegalStateException("Unable to import blocks from archive"); + } + } + repository.saveChanges(); + } + +} diff --git a/src/main/java/org/qortal/utils/ByteArray.java b/src/main/java/org/qortal/utils/ByteArray.java index d89714e14..d3464c9f6 100644 --- a/src/main/java/org/qortal/utils/ByteArray.java +++ b/src/main/java/org/qortal/utils/ByteArray.java @@ -1,12 +1,19 @@ package org.qortal.utils; +import java.util.Arrays; +import java.util.Objects; + public class ByteArray implements Comparable { private int hash; public final byte[] value; public ByteArray(byte[] value) { - this.value = value; + this.value = Objects.requireNonNull(value); + } + + public static ByteArray of(byte[] value) { + return new ByteArray(value); } @Override @@ -14,36 +21,39 @@ public boolean equals(Object other) { if (this == other) return true; - if (other instanceof ByteArray) - return this.compareTo((ByteArray) other) == 0; - if (other instanceof byte[]) - return this.compareTo((byte[]) other) == 0; + return Arrays.equals(this.value, (byte[]) other); + + if (other instanceof ByteArray) + return Arrays.equals(this.value, ((ByteArray) other).value); return false; } @Override public int hashCode() { - int h = hash; - if (h == 0 && value.length > 0) { - byte[] val = value; + int h = this.hash; + byte[] val = this.value; + + if (h == 0 && val.length > 0) { + h = 1; for (int i = 0; i < val.length; ++i) h = 31 * h + val[i]; - hash = h; + this.hash = h; } return h; } @Override public int compareTo(ByteArray other) { - return this.compareTo(other.value); + Objects.requireNonNull(other); + return this.compareToPrimitive(other.value); } - public int compareTo(byte[] otherValue) { - byte[] val = value; + public int compareToPrimitive(byte[] otherValue) { + byte[] val = this.value; if (val.length < otherValue.length) return -1; @@ -63,4 +73,17 @@ public int compareTo(byte[] otherValue) { return 0; } + public String toString() { + StringBuilder sb = new StringBuilder(3 + this.value.length * 6); + sb.append("["); + + if (this.value.length > 0) + sb.append(this.value[0]); + + for (int i = 1; i < this.value.length; ++i) + sb.append(", ").append(this.value[i]); + + return sb.append("]").toString(); + } + } diff --git a/src/main/java/org/qortal/utils/EnumUtils.java b/src/main/java/org/qortal/utils/EnumUtils.java new file mode 100644 index 000000000..9a486b118 --- /dev/null +++ b/src/main/java/org/qortal/utils/EnumUtils.java @@ -0,0 +1,15 @@ +package org.qortal.utils; + +import java.util.Arrays; + +public class EnumUtils { + + public static String[] getNames(Class> e) { + return Arrays.stream(e.getEnumConstants()).map(Enum::name).toArray(String[]::new); + } + + public static String getNames(Class> e, String delimiter) { + return String.join(delimiter, EnumUtils.getNames(e)); + } + +} diff --git a/src/main/java/org/qortal/utils/ExecuteProduceConsume.java b/src/main/java/org/qortal/utils/ExecuteProduceConsume.java index 334322bb5..c639d3641 100644 --- a/src/main/java/org/qortal/utils/ExecuteProduceConsume.java +++ b/src/main/java/org/qortal/utils/ExecuteProduceConsume.java @@ -125,8 +125,8 @@ public void run() { // It's possible this might need to become a class instance private volatile boolean canBlock = false; - while (true) { - final Task task; + while (!Thread.currentThread().isInterrupted()) { + Task task = null; this.logger.trace(() -> String.format("[%d] waiting to produce...", Thread.currentThread().getId())); @@ -142,7 +142,16 @@ public void run() { Thread.currentThread().getId(), this.activeThreadCount, this.consumerCount, lambdaCanIdle)); final long beforeProduce = isLoggerTraceEnabled ? System.currentTimeMillis() : 0; - task = produceTask(canBlock); + + try { + task = produceTask(canBlock); + } catch (InterruptedException e) { + // We're in shutdown situation so exit + Thread.currentThread().interrupt(); + } catch (Exception e) { + this.logger.warn(() -> String.format("[%d] exception while trying to produce task", Thread.currentThread().getId()), e); + } + this.logger.trace(() -> String.format("[%d] producing took %dms", Thread.currentThread().getId(), System.currentTimeMillis() - beforeProduce)); } @@ -155,7 +164,8 @@ public void run() { --this.activeThreadCount; this.logger.trace(() -> String.format("[%d] ending, activeThreadCount now: %d", Thread.currentThread().getId(), this.activeThreadCount)); - break; + + return; } // We're the last surviving thread - producer can afford to block next round @@ -192,7 +202,16 @@ public void run() { } this.logger.trace(() -> String.format("[%d] performing task...", Thread.currentThread().getId())); - task.perform(); // This can block for a while + + try { + task.perform(); // This can block for a while + } catch (InterruptedException e) { + // We're in shutdown situation so exit + Thread.currentThread().interrupt(); + } catch (Exception e) { + this.logger.warn(() -> String.format("[%d] exception while performing task", Thread.currentThread().getId()), e); + } + this.logger.trace(() -> String.format("[%d] finished task", Thread.currentThread().getId())); synchronized (this) { @@ -206,8 +225,6 @@ public void run() { canBlock = false; } } - } catch (InterruptedException e) { - // We're in shutdown situation so exit } finally { if (this.isLoggerTraceEnabled) Thread.currentThread().setName(this.className); diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java new file mode 100644 index 000000000..1b3de544e --- /dev/null +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -0,0 +1,270 @@ +package org.qortal.utils; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.qortal.settings.Settings; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; + +public class FilesystemUtils { + + public static boolean isDirectoryEmpty(Path path) throws IOException { + if (Files.isDirectory(path)) { + try (DirectoryStream directory = Files.newDirectoryStream(path)) { + return !directory.iterator().hasNext(); + } + } + + return false; + } + + public static void copyAndReplaceDirectory(String sourceDirectoryLocation, String destinationDirectoryLocation) throws IOException { + // Ensure parent folders exist in the destination + File destFile = new File(destinationDirectoryLocation); + if (destFile != null) { + destFile.mkdirs(); + } + if (destFile == null || !destFile.exists()) { + throw new IOException("Destination directory doesn't exist"); + } + + // If the destination directory isn't empty, delete its contents + if (!FilesystemUtils.isDirectoryEmpty(destFile.toPath())) { + FileUtils.deleteDirectory(destFile); + destFile.mkdirs(); + } + + Files.walk(Paths.get(sourceDirectoryLocation)) + .forEach(source -> { + Path destination = Paths.get(destinationDirectoryLocation, source.toString() + .substring(sourceDirectoryLocation.length())); + try { + Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + e.printStackTrace(); + } + }); + } + + + /** + * moveFile + * Allows files to be moved between filesystems + * + * @param source + * @param dest + * @param cleanup + * @throws IOException + */ + public static void moveFile(Path source, Path dest, boolean cleanup) throws IOException { + if (source.compareTo(dest) == 0) { + // Source path matches destination path already + return; + } + + File sourceFile = new File(source.toString()); + if (sourceFile == null || !sourceFile.exists()) { + throw new IOException("Source file doesn't exist"); + } + if (!sourceFile.isFile()) { + throw new IOException("Source isn't a file"); + } + + // Ensure parent folders exist in the destination + File destFile = new File(dest.toString()); + File destParentFile = destFile.getParentFile(); + if (destParentFile != null) { + destParentFile.mkdirs(); + } + if (destParentFile == null || !destParentFile.exists()) { + throw new IOException("Destination directory doesn't exist"); + } + + // Copy to destination + Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + + // Delete existing + if (FilesystemUtils.pathInsideDataOrTempPath(source)) { + Files.delete(source); + } + + if (cleanup) { + // ... and delete its parent directory if empty + Path parentDirectory = source.getParent(); + if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) { + Files.deleteIfExists(parentDirectory); + } + } + } + + /** + * moveDirectory + * Allows directories to be moved between filesystems + * + * @param source + * @param dest + * @param cleanup + * @throws IOException + */ + public static void moveDirectory(Path source, Path dest, boolean cleanup) throws IOException { + if (source.compareTo(dest) == 0) { + // Source path matches destination path already + return; + } + + File sourceFile = new File(source.toString()); + File destFile = new File(dest.toString()); + if (sourceFile == null || !sourceFile.exists()) { + throw new IOException("Source directory doesn't exist"); + } + if (!sourceFile.isDirectory()) { + throw new IOException("Source isn't a directory"); + } + + // Ensure parent folders exist in the destination + destFile.mkdirs(); + if (destFile == null || !destFile.exists()) { + throw new IOException("Destination directory doesn't exist"); + } + + // Copy to destination + FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString()); + + // Delete existing + if (FilesystemUtils.pathInsideDataOrTempPath(source)) { + File directory = new File(source.toString()); + System.out.println(String.format("Deleting directory %s", directory.toString())); + FileUtils.deleteDirectory(directory); + } + + if (cleanup) { + // ... and delete its parent directory if empty + Path parentDirectory = source.getParent(); + if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) { + Files.deleteIfExists(parentDirectory); + } + } + } + + public static boolean safeDeleteDirectory(Path path, boolean cleanup) throws IOException { + boolean success = false; + + // Delete path, if it exists in our data/temp directory + if (FilesystemUtils.pathInsideDataOrTempPath(path)) { + if (Files.exists(path)) { + File directory = new File(path.toString()); + FileUtils.deleteDirectory(directory); + success = true; + } + } + + if (success && cleanup) { + // Delete the parent directories if they are empty (and exist in our data/temp directory) + FilesystemUtils.safeDeleteEmptyParentDirectories(path); + } + + return success; + } + + public static void safeDeleteEmptyParentDirectories(Path path) throws IOException { + final Path parentPath = path.toAbsolutePath().getParent(); + if (!parentPath.toFile().isDirectory()) { + return; + } + if (!FilesystemUtils.pathInsideDataOrTempPath(parentPath)) { + return; + } + try { + Files.deleteIfExists(parentPath); + + } catch (DirectoryNotEmptyException e) { + // We've reached the limits of what we can delete + return; + } + + FilesystemUtils.safeDeleteEmptyParentDirectories(parentPath); + } + + public static boolean pathInsideDataOrTempPath(Path path) { + if (path == null) { + return false; + } + Path dataPath = Paths.get(Settings.getInstance().getDataPath()).toAbsolutePath(); + Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath()).toAbsolutePath(); + Path absolutePath = path.toAbsolutePath(); + if (absolutePath.startsWith(dataPath) || absolutePath.startsWith(tempDataPath)) { + return true; + } + return false; + } + + public static boolean isChild(Path child, Path parent) { + return child.toAbsolutePath().startsWith(parent.toAbsolutePath()); + } + + public static long getDirectorySize(Path path) throws IOException { + if (path == null || !Files.exists(path)) { + return 0L; + } + return Files.walk(path) + .filter(p -> p.toFile().isFile()) + .mapToLong(p -> p.toFile().length()) + .sum(); + } + + + /** + * getSingleFileContents + * Return the content of the file at given path. + * If the path is a directory, the contents will be returned + * only if it contains a single file. + * + * @param path + * @return + * @throws IOException + */ + public static byte[] getSingleFileContents(Path path) throws IOException { + byte[] data = null; + // TODO: limit the file size that can be loaded into memory + + // If the path is a file, read the contents directly + if (path.toFile().isFile()) { + data = Files.readAllBytes(path); + } + + // Or if it's a directory, only load file contents if there is a single file inside it + else if (path.toFile().isDirectory()) { + String[] files = ArrayUtils.removeElement(path.toFile().list(), ".qortal"); + if (files.length == 1) { + Path filePath = Paths.get(path.toString(), files[0]); + data = Files.readAllBytes(filePath); + } + } + + return data; + } + + public static byte[] readFromFile(String filePath, long position, int size) throws IOException { + RandomAccessFile file = new RandomAccessFile(filePath, "r"); + file.seek(position); + byte[] bytes = new byte[size]; + file.read(bytes); + file.close(); + return bytes; + } + + public static String readUtf8StringFromFile(String filePath, long position, int size) throws IOException { + return new String(FilesystemUtils.readFromFile(filePath, position, size), StandardCharsets.UTF_8); + } + + public static boolean fileEndsWithNewline(Path path) throws IOException { + long length = Files.size(path); + String lastCharacter = FilesystemUtils.readUtf8StringFromFile(path.toString(), length-1, 1); + return (lastCharacter.equals("\n") || lastCharacter.equals("\r")); + } + +} diff --git a/src/main/java/org/qortal/utils/LoggingUtils.java b/src/main/java/org/qortal/utils/LoggingUtils.java new file mode 100644 index 000000000..eeb2a7c06 --- /dev/null +++ b/src/main/java/org/qortal/utils/LoggingUtils.java @@ -0,0 +1,28 @@ +package org.qortal.utils; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class LoggingUtils { + + public static void fixLegacyLog4j2Properties() { + Path log4j2PropertiesPath = Paths.get("log4j2.properties"); + if (Files.exists(log4j2PropertiesPath)) { + try { + String content = FileUtils.readFileToString(log4j2PropertiesPath.toFile(), "UTF-8"); + if (content.contains("${dirname:-}")) { + content = content.replace("${dirname:-}", "./"); + FileUtils.writeStringToFile(log4j2PropertiesPath.toFile(), content, "UTF-8"); + } + } catch (IOException e) { + // Not much we can do here + } + } + } + +} diff --git a/src/main/java/org/qortal/utils/NTP.java b/src/main/java/org/qortal/utils/NTP.java index b141ed962..779db41b7 100644 --- a/src/main/java/org/qortal/utils/NTP.java +++ b/src/main/java/org/qortal/utils/NTP.java @@ -28,7 +28,8 @@ public class NTP implements Runnable { private static volatile boolean isStopping = false; private static ExecutorService instanceExecutor; private static NTP instance; - private static volatile Long offset = null; + private static volatile boolean isOffsetSet = false; + private static volatile long offset = 0; static class NTPServer { private static final int MIN_POLL = 64; @@ -136,6 +137,7 @@ public static void shutdownNow() { public static synchronized void setFixedOffset(Long offset) { // Fix offset, e.g. for testing NTP.offset = offset; + isOffsetSet = true; } /** @@ -144,7 +146,7 @@ public static synchronized void setFixedOffset(Long offset) { * @return internet time (ms), or null if unsynchronized. */ public static Long getTime() { - if (NTP.offset == null) + if (!isOffsetSet) return null; return System.currentTimeMillis() + NTP.offset; @@ -248,6 +250,7 @@ private void calculateOffset() { thresholdStddev, filteredMean, filteredStddev, numberValues, ntpServers.size())); NTP.offset = (long) filteredMean; + isOffsetSet = true; LOGGER.debug(() -> String.format("New NTP offset: %d", NTP.offset)); } } diff --git a/src/main/java/org/qortal/utils/Serialization.java b/src/main/java/org/qortal/utils/Serialization.java index e9bf6e0ee..268884776 100644 --- a/src/main/java/org/qortal/utils/Serialization.java +++ b/src/main/java/org/qortal/utils/Serialization.java @@ -100,12 +100,26 @@ public static byte[] deserializePublicKey(ByteBuffer byteBuffer) { return bytes; } + /** + * Original serializeSizedString() method used in various transaction types + * @param bytes + * @param string + * @throws UnsupportedEncodingException + * @throws IOException + */ public static void serializeSizedString(ByteArrayOutputStream bytes, String string) throws UnsupportedEncodingException, IOException { byte[] stringBytes = string.getBytes(StandardCharsets.UTF_8); bytes.write(Ints.toByteArray(stringBytes.length)); bytes.write(stringBytes); } + /** + * Original deserializeSizedString() method used in various transaction types + * @param byteBuffer + * @param maxSize + * @return + * @throws TransformationException + */ public static String deserializeSizedString(ByteBuffer byteBuffer, int maxSize) throws TransformationException { int size = byteBuffer.getInt(); if (size > maxSize) @@ -120,4 +134,54 @@ public static String deserializeSizedString(ByteBuffer byteBuffer, int maxSize) return new String(bytes, StandardCharsets.UTF_8); } + /** + * Alternate version of serializeSizedString() added for ARBITRARY transactions. + * These two methods can ultimately be merged together once unit tests can + * confirm that they process data identically. + * @param bytes + * @param string + * @throws UnsupportedEncodingException + * @throws IOException + */ + public static void serializeSizedStringV2(ByteArrayOutputStream bytes, String string) throws UnsupportedEncodingException, IOException { + byte[] stringBytes = null; + int stringBytesLength = 0; + + if (string != null) { + stringBytes = string.getBytes(StandardCharsets.UTF_8); + stringBytesLength = stringBytes.length; + } + bytes.write(Ints.toByteArray(stringBytesLength)); + if (stringBytesLength > 0) { + bytes.write(stringBytes); + } + } + + /** + * Alternate version of serializeSizedString() added for ARBITRARY transactions. + * The main difference is that blank strings are returned as null. + * These two methods can ultimately be merged together once any differences are + * solved, and unit tests can confirm that they process data identically. + * @param byteBuffer + * @param maxSize + * @return + * @throws TransformationException + */ + public static String deserializeSizedStringV2(ByteBuffer byteBuffer, int maxSize) throws TransformationException { + int size = byteBuffer.getInt(); + if (size > maxSize) + throw new TransformationException("Serialized string too long"); + + if (size > byteBuffer.remaining()) + throw new TransformationException("Byte data too short for serialized string"); + + if (size == 0) + return null; + + byte[] bytes = new byte[size]; + byteBuffer.get(bytes); + + return new String(bytes, StandardCharsets.UTF_8); + } + } diff --git a/src/main/java/org/qortal/utils/SevenZ.java b/src/main/java/org/qortal/utils/SevenZ.java new file mode 100644 index 000000000..5126b292b --- /dev/null +++ b/src/main/java/org/qortal/utils/SevenZ.java @@ -0,0 +1,85 @@ +// +// Code originally written by memorynotfound +// https://memorynotfound.com/java-7z-seven-zip-example-compress-decompress-file/ +// Modified Sept 2021 by Qortal Core dev team +// + +package org.qortal.utils; + +import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry; +import org.apache.commons.compress.archivers.sevenz.SevenZFile; +import org.apache.commons.compress.archivers.sevenz.SevenZOutputFile; +import org.qortal.gui.SplashFrame; + +import java.io.*; + +public class SevenZ { + + private SevenZ() { + + } + + public static void compress(String outputPath, File... files) throws IOException { + try (SevenZOutputFile out = new SevenZOutputFile(new File(outputPath))){ + for (File file : files){ + addToArchiveCompression(out, file, "."); + } + } + } + + public static void decompress(String in, File destination) throws IOException { + SevenZFile sevenZFile = new SevenZFile(new File(in)); + SevenZArchiveEntry entry; + while ((entry = sevenZFile.getNextEntry()) != null){ + if (entry.isDirectory()){ + continue; + } + File curfile = new File(destination, entry.getName()); + File parent = curfile.getParentFile(); + if (!parent.exists()) { + parent.mkdirs(); + } + long fileSize = entry.getSize(); + + FileOutputStream out = new FileOutputStream(curfile); + byte[] b = new byte[1024 * 1024]; + int count; + long extracted = 0; + + while ((count = sevenZFile.read(b)) > 0) { + out.write(b, 0, count); + extracted += count; + + int progress = (int)((double)extracted / (double)fileSize * 100); + SplashFrame.getInstance().updateStatus(String.format("Extracting %s... (%d%%)", curfile.getName(), progress)); + } + out.close(); + } + } + + private static void addToArchiveCompression(SevenZOutputFile out, File file, String dir) throws IOException { + String name = dir + File.separator + file.getName(); + if (file.isFile()){ + SevenZArchiveEntry entry = out.createArchiveEntry(file, name); + out.putArchiveEntry(entry); + + FileInputStream in = new FileInputStream(file); + byte[] b = new byte[8192]; + int count = 0; + while ((count = in.read(b)) > 0) { + out.write(b, 0, count); + } + out.closeArchiveEntry(); + + } else if (file.isDirectory()) { + File[] children = file.listFiles(); + if (children != null){ + for (File child : children){ + addToArchiveCompression(out, child, name); + } + } + } else { + System.out.println(file.getName() + " is not supported"); + } + } +} diff --git a/src/main/java/org/qortal/utils/Triple.java b/src/main/java/org/qortal/utils/Triple.java index 5095a2dae..0b9757ee2 100644 --- a/src/main/java/org/qortal/utils/Triple.java +++ b/src/main/java/org/qortal/utils/Triple.java @@ -1,42 +1,55 @@ package org.qortal.utils; -public class Triple { +public class Triple { - private T a; - private U b; - private V c; + @FunctionalInterface + public interface TripleConsumer { + public void accept(A a, B b, C c); + } + + private A a; + private B b; + private C c; + + public static Triple valueOf(A a, B b, C c) { + return new Triple<>(a, b, c); + } public Triple() { } - public Triple(T a, U b, V c) { + public Triple(A a, B b, C c) { this.a = a; this.b = b; this.c = c; } - public void setA(T a) { + public void setA(A a) { this.a = a; } - public T getA() { + public A getA() { return a; } - public void setB(U b) { + public void setB(B b) { this.b = b; } - public U getB() { + public B getB() { return b; } - public void setC(V c) { + public void setC(C c) { this.c = c; } - public V getC() { + public C getC() { return c; } + public void consume(TripleConsumer consumer) { + consumer.accept(this.a, this.b, this.c); + } + } diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java new file mode 100644 index 000000000..c61723e75 --- /dev/null +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -0,0 +1,139 @@ +/* + * MIT License + * + * Copyright (c) 2017 Eugen Paraschiv + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Code modified in 2021 for Qortal Core + * + */ + +package org.qortal.utils; + +import org.qortal.controller.Controller; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +public class ZipUtils { + + public static void zip(String sourcePath, String destFilePath, String enclosingFolderName) throws IOException, InterruptedException { + File sourceFile = new File(sourcePath); + boolean isSingleFile = Paths.get(sourcePath).toFile().isFile(); + FileOutputStream fileOutputStream = new FileOutputStream(destFilePath); + ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); + ZipUtils.zip(sourceFile, enclosingFolderName, zipOutputStream, isSingleFile); + zipOutputStream.close(); + fileOutputStream.close(); + } + + public static void zip(final File fileToZip, final String enclosingFolderName, final ZipOutputStream zipOut, boolean isSingleFile) throws IOException, InterruptedException { + if (Controller.isStopping()) { + throw new InterruptedException("Controller is stopping"); + } + + // Handle single file resources slightly differently + if (isSingleFile) { + // Create enclosing folder + zipOut.putNextEntry(new ZipEntry(enclosingFolderName + "/")); + zipOut.closeEntry(); + // Place the supplied file within the folder + ZipUtils.zip(fileToZip, enclosingFolderName + "/" + fileToZip.getName(), zipOut, false); + return; + } + + if (fileToZip.isDirectory()) { + if (enclosingFolderName.endsWith("/")) { + zipOut.putNextEntry(new ZipEntry(enclosingFolderName)); + zipOut.closeEntry(); + } else { + zipOut.putNextEntry(new ZipEntry(enclosingFolderName + "/")); + zipOut.closeEntry(); + } + final File[] children = fileToZip.listFiles(); + for (final File childFile : children) { + ZipUtils.zip(childFile, enclosingFolderName + "/" + childFile.getName(), zipOut, false); + } + return; + } + final FileInputStream fis = new FileInputStream(fileToZip); + final ZipEntry zipEntry = new ZipEntry(enclosingFolderName); + zipOut.putNextEntry(zipEntry); + final byte[] bytes = new byte[1024]; + int length; + while ((length = fis.read(bytes)) >= 0) { + zipOut.write(bytes, 0, length); + } + fis.close(); + } + + public static void unzip(String sourcePath, String destPath) throws IOException { + final File destDir = new File(destPath); + final byte[] buffer = new byte[1024]; + final ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath)); + ZipEntry zipEntry = zis.getNextEntry(); + while (zipEntry != null) { + final File newFile = ZipUtils.newFile(destDir, zipEntry); + if (zipEntry.isDirectory()) { + if (!newFile.isDirectory() && !newFile.mkdirs()) { + throw new IOException("Failed to create directory " + newFile); + } + } else { + File parent = newFile.getParentFile(); + if (!parent.isDirectory() && !parent.mkdirs()) { + throw new IOException("Failed to create directory " + parent); + } + + final FileOutputStream fos = new FileOutputStream(newFile); + int len; + while ((len = zis.read(buffer)) > 0) { + fos.write(buffer, 0, len); + } + fos.close(); + } + zipEntry = zis.getNextEntry(); + } + zis.closeEntry(); + zis.close(); + } + + /** + * See: https://snyk.io/research/zip-slip-vulnerability + */ + public static File newFile(File destinationDir, ZipEntry zipEntry) throws IOException { + File destFile = new File(destinationDir, zipEntry.getName()); + + String destDirPath = destinationDir.getCanonicalPath(); + String destFilePath = destFile.getCanonicalPath(); + + if (!destFilePath.startsWith(destDirPath + File.separator)) { + throw new IOException("Entry is outside of the target dir: " + zipEntry.getName()); + } + + return destFile; + } + +} diff --git a/src/main/resources/BIP39/wordlist_en.txt b/src/main/resources/BIP39/wordlist_en.txt deleted file mode 100644 index 942040ed5..000000000 --- a/src/main/resources/BIP39/wordlist_en.txt +++ /dev/null @@ -1,2048 +0,0 @@ -abandon -ability -able -about -above -absent -absorb -abstract -absurd -abuse -access -accident -account -accuse -achieve -acid -acoustic -acquire -across -act -action -actor -actress -actual -adapt -add -addict -address -adjust -admit -adult -advance -advice -aerobic -affair -afford -afraid -again -age -agent -agree -ahead -aim -air -airport -aisle -alarm -album -alcohol -alert -alien -all -alley -allow -almost -alone -alpha -already -also -alter -always -amateur -amazing -among -amount -amused -analyst -anchor -ancient -anger -angle -angry -animal -ankle -announce -annual -another -answer -antenna -antique -anxiety -any -apart -apology -appear -apple -approve -april -arch -arctic -area -arena -argue -arm -armed -armor -army -around -arrange -arrest -arrive -arrow -art -artefact -artist -artwork -ask -aspect -assault -asset -assist -assume -asthma -athlete -atom -attack -attend -attitude -attract -auction -audit -august -aunt -author -auto -autumn -average -avocado -avoid -awake -aware -away -awesome -awful -awkward -axis -baby -bachelor -bacon -badge -bag -balance -balcony -ball -bamboo -banana -banner -bar -barely -bargain -barrel -base -basic -basket -battle -beach -bean -beauty -because -become -beef -before -begin -behave -behind -believe -below -belt -bench -benefit -best -betray -better -between -beyond -bicycle -bid -bike -bind -biology -bird -birth -bitter -black -blade -blame -blanket -blast -bleak -bless -blind -blood -blossom -blouse -blue -blur -blush -board -boat -body -boil -bomb -bone -bonus -book -boost -border -boring -borrow -boss -bottom -bounce -box -boy -bracket -brain -brand -brass -brave -bread -breeze -brick -bridge -brief -bright -bring -brisk -broccoli -broken -bronze -broom -brother -brown -brush -bubble -buddy -budget -buffalo -build -bulb -bulk -bullet -bundle -bunker -burden -burger -burst -bus -business -busy -butter -buyer -buzz -cabbage -cabin -cable -cactus -cage -cake -call -calm -camera -camp -can -canal -cancel -candy -cannon -canoe -canvas -canyon -capable -capital -captain -car -carbon -card -cargo -carpet -carry -cart -case -cash -casino -castle -casual -cat -catalog -catch -category -cattle -caught -cause -caution -cave -ceiling -celery -cement -census -century -cereal -certain -chair -chalk -champion -change -chaos -chapter -charge -chase -chat -cheap -check -cheese -chef -cherry -chest -chicken -chief -child -chimney -choice -choose -chronic -chuckle -chunk -churn -cigar -cinnamon -circle -citizen -city -civil -claim -clap -clarify -claw -clay -clean -clerk -clever -click -client -cliff -climb -clinic -clip -clock -clog -close -cloth -cloud -clown -club -clump -cluster -clutch -coach -coast -coconut -code -coffee -coil -coin -collect -color -column -combine -come -comfort -comic -common -company -concert -conduct -confirm -congress -connect -consider -control -convince -cook -cool -copper -copy -coral -core -corn -correct -cost -cotton -couch -country -couple -course -cousin -cover -coyote -crack -cradle -craft -cram -crane -crash -crater -crawl -crazy -cream -credit -creek -crew -cricket -crime -crisp -critic -crop -cross -crouch -crowd -crucial -cruel -cruise -crumble -crunch -crush -cry -crystal -cube -culture -cup -cupboard -curious -current -curtain -curve -cushion -custom -cute -cycle -dad -damage -damp -dance -danger -daring -dash -daughter -dawn -day -deal -debate -debris -decade -december -decide -decline -decorate -decrease -deer -defense -define -defy -degree -delay -deliver -demand -demise -denial -dentist -deny -depart -depend -deposit -depth -deputy -derive -describe -desert -design -desk -despair -destroy -detail -detect -develop -device -devote -diagram -dial -diamond -diary -dice -diesel -diet -differ -digital -dignity -dilemma -dinner -dinosaur -direct -dirt -disagree -discover -disease -dish -dismiss -disorder -display -distance -divert -divide -divorce -dizzy -doctor -document -dog -doll -dolphin -domain -donate -donkey -donor -door -dose -double -dove -draft -dragon -drama -drastic -draw -dream -dress -drift -drill -drink -drip -drive -drop -drum -dry -duck -dumb -dune -during -dust -dutch -duty -dwarf -dynamic -eager -eagle -early -earn -earth -easily -east -easy -echo -ecology -economy -edge -edit -educate -effort -egg -eight -either -elbow -elder -electric -elegant -element -elephant -elevator -elite -else -embark -embody -embrace -emerge -emotion -employ -empower -empty -enable -enact -end -endless -endorse -enemy -energy -enforce -engage -engine -enhance -enjoy -enlist -enough -enrich -enroll -ensure -enter -entire -entry -envelope -episode -equal -equip -era -erase -erode -erosion -error -erupt -escape -essay -essence -estate -eternal -ethics -evidence -evil -evoke -evolve -exact -example -excess -exchange -excite -exclude -excuse -execute -exercise -exhaust -exhibit -exile -exist -exit -exotic -expand -expect -expire -explain -expose -express -extend -extra -eye -eyebrow -fabric -face -faculty -fade -faint -faith -fall -false -fame -family -famous -fan -fancy -fantasy -farm -fashion -fat -fatal -father -fatigue -fault -favorite -feature -february -federal -fee -feed -feel -female -fence -festival -fetch -fever -few -fiber -fiction -field -figure -file -film -filter -final -find -fine -finger -finish -fire -firm -first -fiscal -fish -fit -fitness -fix -flag -flame -flash -flat -flavor -flee -flight -flip -float -flock -floor -flower -fluid -flush -fly -foam -focus -fog -foil -fold -follow -food -foot -force -forest -forget -fork -fortune -forum -forward -fossil -foster -found -fox -fragile -frame -frequent -fresh -friend -fringe -frog -front -frost -frown -frozen -fruit -fuel -fun -funny -furnace -fury -future -gadget -gain -galaxy -gallery -game -gap -garage -garbage -garden -garlic -garment -gas -gasp -gate -gather -gauge -gaze -general -genius -genre -gentle -genuine -gesture -ghost -giant -gift -giggle -ginger -giraffe -girl -give -glad -glance -glare -glass -glide -glimpse -globe -gloom -glory -glove -glow -glue -goat -goddess -gold -good -goose -gorilla -gospel -gossip -govern -gown -grab -grace -grain -grant -grape -grass -gravity -great -green -grid -grief -grit -grocery -group -grow -grunt -guard -guess -guide -guilt -guitar -gun -gym -habit -hair -half -hammer -hamster -hand -happy -harbor -hard -harsh -harvest -hat -have -hawk -hazard -head -health -heart -heavy -hedgehog -height -hello -helmet -help -hen -hero -hidden -high -hill -hint -hip -hire -history -hobby -hockey -hold -hole -holiday -hollow -home -honey -hood -hope -horn -horror -horse -hospital -host -hotel -hour -hover -hub -huge -human -humble -humor -hundred -hungry -hunt -hurdle -hurry -hurt -husband -hybrid -ice -icon -idea -identify -idle -ignore -ill -illegal -illness -image -imitate -immense -immune -impact -impose -improve -impulse -inch -include -income -increase -index -indicate -indoor -industry -infant -inflict -inform -inhale -inherit -initial -inject -injury -inmate -inner -innocent -input -inquiry -insane -insect -inside -inspire -install -intact -interest -into -invest -invite -involve -iron -island -isolate -issue -item -ivory -jacket -jaguar -jar -jazz -jealous -jeans -jelly -jewel -job -join -joke -journey -joy -judge -juice -jump -jungle -junior -junk -just -kangaroo -keen -keep -ketchup -key -kick -kid -kidney -kind -kingdom -kiss -kit -kitchen -kite -kitten -kiwi -knee -knife -knock -know -lab -label -labor -ladder -lady -lake -lamp -language -laptop -large -later -latin -laugh -laundry -lava -law -lawn -lawsuit -layer -lazy -leader -leaf -learn -leave -lecture -left -leg -legal -legend -leisure -lemon -lend -length -lens -leopard -lesson -letter -level -liar -liberty -library -license -life -lift -light -like -limb -limit -link -lion -liquid -list -little -live -lizard -load -loan -lobster -local -lock -logic -lonely -long -loop -lottery -loud -lounge -love -loyal -lucky -luggage -lumber -lunar -lunch -luxury -lyrics -machine -mad -magic -magnet -maid -mail -main -major -make -mammal -man -manage -mandate -mango -mansion -manual -maple -marble -march -margin -marine -market -marriage -mask -mass -master -match -material -math -matrix -matter -maximum -maze -meadow -mean -measure -meat -mechanic -medal -media -melody -melt -member -memory -mention -menu -mercy -merge -merit -merry -mesh -message -metal -method -middle -midnight -milk -million -mimic -mind -minimum -minor -minute -miracle -mirror -misery -miss -mistake -mix -mixed -mixture -mobile -model -modify -mom -moment -monitor -monkey -monster -month -moon -moral -more -morning -mosquito -mother -motion -motor -mountain -mouse -move -movie -much -muffin -mule -multiply -muscle -museum -mushroom -music -must -mutual -myself -mystery -myth -naive -name -napkin -narrow -nasty -nation -nature -near -neck -need -negative -neglect -neither -nephew -nerve -nest -net -network -neutral -never -news -next -nice -night -noble -noise -nominee -noodle -normal -north -nose -notable -note -nothing -notice -novel -now -nuclear -number -nurse -nut -oak -obey -object -oblige -obscure -observe -obtain -obvious -occur -ocean -october -odor -off -offer -office -often -oil -okay -old -olive -olympic -omit -once -one -onion -online -only -open -opera -opinion -oppose -option -orange -orbit -orchard -order -ordinary -organ -orient -original -orphan -ostrich -other -outdoor -outer -output -outside -oval -oven -over -own -owner -oxygen -oyster -ozone -pact -paddle -page -pair -palace -palm -panda -panel -panic -panther -paper -parade -parent -park -parrot -party -pass -patch -path -patient -patrol -pattern -pause -pave -payment -peace -peanut -pear -peasant -pelican -pen -penalty -pencil -people -pepper -perfect -permit -person -pet -phone -photo -phrase -physical -piano -picnic -picture -piece -pig -pigeon -pill -pilot -pink -pioneer -pipe -pistol -pitch -pizza -place -planet -plastic -plate -play -please -pledge -pluck -plug -plunge -poem -poet -point -polar -pole -police -pond -pony -pool -popular -portion -position -possible -post -potato -pottery -poverty -powder -power -practice -praise -predict -prefer -prepare -present -pretty -prevent -price -pride -primary -print -priority -prison -private -prize -problem -process -produce -profit -program -project -promote -proof -property -prosper -protect -proud -provide -public -pudding -pull -pulp -pulse -pumpkin -punch -pupil -puppy -purchase -purity -purpose -purse -push -put -puzzle -pyramid -quality -quantum -quarter -question -quick -quit -quiz -quote -rabbit -raccoon -race -rack -radar -radio -rail -rain -raise -rally -ramp -ranch -random -range -rapid -rare -rate -rather -raven -raw -razor -ready -real -reason -rebel -rebuild -recall -receive -recipe -record -recycle -reduce -reflect -reform -refuse -region -regret -regular -reject -relax -release -relief -rely -remain -remember -remind -remove -render -renew -rent -reopen -repair -repeat -replace -report -require -rescue -resemble -resist -resource -response -result -retire -retreat -return -reunion -reveal -review -reward -rhythm -rib -ribbon -rice -rich -ride -ridge -rifle -right -rigid -ring -riot -ripple -risk -ritual -rival -river -road -roast -robot -robust -rocket -romance -roof -rookie -room -rose -rotate -rough -round -route -royal -rubber -rude -rug -rule -run -runway -rural -sad -saddle -sadness -safe -sail -salad -salmon -salon -salt -salute -same -sample -sand -satisfy -satoshi -sauce -sausage -save -say -scale -scan -scare -scatter -scene -scheme -school -science -scissors -scorpion -scout -scrap -screen -script -scrub -sea -search -season -seat -second -secret -section -security -seed -seek -segment -select -sell -seminar -senior -sense -sentence -series -service -session -settle -setup -seven -shadow -shaft -shallow -share -shed -shell -sheriff -shield -shift -shine -ship -shiver -shock -shoe -shoot -shop -short -shoulder -shove -shrimp -shrug -shuffle -shy -sibling -sick -side -siege -sight -sign -silent -silk -silly -silver -similar -simple -since -sing -siren -sister -situate -six -size -skate -sketch -ski -skill -skin -skirt -skull -slab -slam -sleep -slender -slice -slide -slight -slim -slogan -slot -slow -slush -small -smart -smile -smoke -smooth -snack -snake -snap -sniff -snow -soap -soccer -social -sock -soda -soft -solar -soldier -solid -solution -solve -someone -song -soon -sorry -sort -soul -sound -soup -source -south -space -spare -spatial -spawn -speak -special -speed -spell -spend -sphere -spice -spider -spike -spin -spirit -split -spoil -sponsor -spoon -sport -spot -spray -spread -spring -spy -square -squeeze -squirrel -stable -stadium -staff -stage -stairs -stamp -stand -start -state -stay -steak -steel -stem -step -stereo -stick -still -sting -stock -stomach -stone -stool -story -stove -strategy -street -strike -strong -struggle -student -stuff -stumble -style -subject -submit -subway -success -such -sudden -suffer -sugar -suggest -suit -summer -sun -sunny -sunset -super -supply -supreme -sure -surface -surge -surprise -surround -survey -suspect -sustain -swallow -swamp -swap -swarm -swear -sweet -swift -swim -swing -switch -sword -symbol -symptom -syrup -system -table -tackle -tag -tail -talent -talk -tank -tape -target -task -taste -tattoo -taxi -teach -team -tell -ten -tenant -tennis -tent -term -test -text -thank -that -theme -then -theory -there -they -thing -this -thought -three -thrive -throw -thumb -thunder -ticket -tide -tiger -tilt -timber -time -tiny -tip -tired -tissue -title -toast -tobacco -today -toddler -toe -together -toilet -token -tomato -tomorrow -tone -tongue -tonight -tool -tooth -top -topic -topple -torch -tornado -tortoise -toss -total -tourist -toward -tower -town -toy -track -trade -traffic -tragic -train -transfer -trap -trash -travel -tray -treat -tree -trend -trial -tribe -trick -trigger -trim -trip -trophy -trouble -truck -true -truly -trumpet -trust -truth -try -tube -tuition -tumble -tuna -tunnel -turkey -turn -turtle -twelve -twenty -twice -twin -twist -two -type -typical -ugly -umbrella -unable -unaware -uncle -uncover -under -undo -unfair -unfold -unhappy -uniform -unique -unit -universe -unknown -unlock -until -unusual -unveil -update -upgrade -uphold -upon -upper -upset -urban -urge -usage -use -used -useful -useless -usual -utility -vacant -vacuum -vague -valid -valley -valve -van -vanish -vapor -various -vast -vault -vehicle -velvet -vendor -venture -venue -verb -verify -version -very -vessel -veteran -viable -vibrant -vicious -victory -video -view -village -vintage -violin -virtual -virus -visa -visit -visual -vital -vivid -vocal -voice -void -volcano -volume -vote -voyage -wage -wagon -wait -walk -wall -walnut -want -warfare -warm -warrior -wash -wasp -waste -water -wave -way -wealth -weapon -wear -weasel -weather -web -wedding -weekend -weird -welcome -west -wet -whale -what -wheat -wheel -when -where -whip -whisper -wide -width -wife -wild -will -win -window -wine -wing -wink -winner -winter -wire -wisdom -wise -wish -witness -wolf -woman -wonder -wood -wool -word -work -world -worry -worth -wrap -wreck -wrestle -wrist -write -wrong -yard -year -yellow -you -young -youth -zebra -zero -zone -zoo diff --git a/src/main/resources/block-212937-deltas.json b/src/main/resources/block-212937-deltas.json new file mode 100644 index 000000000..7e6a9d427 --- /dev/null +++ b/src/main/resources/block-212937-deltas.json @@ -0,0 +1,719 @@ +[ + { "address": "Qa43JP9hnNjfSy1f3LYYNFhhSuMokUoYqQ", "assetId": 0, "balance": 0.00003628}, + { "address": "Qa4VMxDhmGH5dgYLiuFSyaWju8xb2fGZhs", "assetId": 0, "balance": -0.00010321}, + { "address": "Qa8pRawmcviX1BHQpNCt4vBYHz7HjdNfkL", "assetId": 0, "balance": -0.00010321}, + { "address": "QacFHzkV265jd57jfTZ5gSuW8dj4W1ttYs", "assetId": 0, "balance": 0.00003628}, + { "address": "QadVAZb3yc78yjQwQDJ8bXCvFohAdEovu7", "assetId": 0, "balance": -0.00010321}, + { "address": "QaeFXCfi73Wptwve5R2RdFSUJUs2dqsHXY", "assetId": 0, "balance": 0.00003628}, + { "address": "Qaf5BHpXrWKK3dprQX7zcCXtCPiQdhm7oo", "assetId": 0, "balance": 0.00001276}, + { "address": "QaK6URQq4vwEDWyBtmS25kor49Z56An7xn", "assetId": 0, "balance": -0.00010321}, + { "address": "QakzYX9JZyUjRYtXJeaQbirXTuUqMFdp7d", "assetId": 0, "balance": -0.00010321}, + { "address": "QaLWoAkjc7ip5Y38p5FX8vVbEYFHCz2zHh", "assetId": 0, "balance": -0.00010321}, + { "address": "QanYq81HNrintpSE6FPRVioHfhCHzTkN83", "assetId": 0, "balance": -0.00010321}, + { "address": "QaoVMdbpPQDHfBAar1HfPoEBzDoFa2PjS8", "assetId": 0, "balance": -0.00010321}, + { "address": "QapE6pVuceYdVKnHVePbeaqf5QNeoDKbqr", "assetId": 0, "balance": 0.00001276}, + { "address": "QaPKuyyQtXJcsVhKLKgxCcYewxwaawxLrB", "assetId": 0, "balance": 0.00000011}, + { "address": "QaPKuyyQtXJcsVhKLKgxCcYewxwaawxLrB", "assetId": 2, "balance": 0.00000011}, + { "address": "Qaq4sV9wkdSSSgJtaPuuSV15tqsVPD78rz", "assetId": 0, "balance": 0.00003628}, + { "address": "Qar2d2pXBP7NCe8mNTbzTSzAQ48ptGqrMC", "assetId": 0, "balance": 0.00003628}, + { "address": "QasrD9TxGAuWqnRpJxBBwwh7Nj6BHiA77d", "assetId": 0, "balance": 0.00003628}, + { "address": "QaSXJSHbQ4xmwaQd5tKJMAqvVzwzLvhddP", "assetId": 0, "balance": 0.00003628}, + { "address": "Qata5oApMShnD4F1kcgSJMTiYsxTPSFW4F", "assetId": 0, "balance": 0.00001283}, + { "address": "QaUciVnbQDXdNygJadEY31PuDEBLi6Spmu", "assetId": 0, "balance": 0.00000011}, + { "address": "QaUciVnbQDXdNygJadEY31PuDEBLi6Spmu", "assetId": 2, "balance": 0.00000011}, + { "address": "QavBkY3kRPJxtvsU5yWuhUnMdDWvs4E3Dw", "assetId": 0, "balance": 0.00003628}, + { "address": "QavzMF32Xvbg4Q4rM9a9R5WVmQZt7iW4Fa", "assetId": 0, "balance": 0.00003628}, + { "address": "QawB5MesBratjs2d9EMnXnrN4EC7gw7LRw", "assetId": 0, "balance": 0.00000015}, + { "address": "QawB5MesBratjs2d9EMnXnrN4EC7gw7LRw", "assetId": 2, "balance": 0.00000004}, + { "address": "QawSgZ7i2LLFTKyPxQptk9gN526ihy5yZi", "assetId": 0, "balance": 0.00003628}, + { "address": "QaxMTV7fGSnibyvVjRaX7FRerrb9aMW6SR", "assetId": 0, "balance": 0.00003628}, + { "address": "QaywytB5dqQoDgBejmEhWXWaDgfL1CRBGH", "assetId": 0, "balance": 0.00003628}, + { "address": "QaZs97g4Mbq9tXMoBWbhw3jFvBBVkWKS5F", "assetId": 0, "balance": 0.00003628}, + { "address": "Qb1pkXG4xufNS3ki354CWkEmC1gmz6D2H7", "assetId": 0, "balance": 0.00003628}, + { "address": "Qb2VbWdrY2E9uLALmyan35E6H5ze6tBmxX", "assetId": 0, "balance": 0.00003628}, + { "address": "Qb5F6KX4Fg1LRM21QJF48m1EYxnipFfRy1", "assetId": 0, "balance": 0.00003628}, + { "address": "Qb9Ycc3f6KUyWPMBGgeEczy4HorPFfy2hj", "assetId": 0, "balance": -0.00010321}, + { "address": "QbBMsJvjo4ZouPPGegxRs5kqKuRzfLRYMU", "assetId": 0, "balance": 0.00001276}, + { "address": "QbchhqR3QLE1T1kRzySFWsVamhPy8oyeC1", "assetId": 0, "balance": -0.00010321}, + { "address": "Qbcy4uyMkQF2JXYqGkueDiFNZ4tHjRg8CR", "assetId": 0, "balance": 0.00000011}, + { "address": "QbdV2vipqMui1eQnj9ZudQgw4e8zRgQ9Lk", "assetId": 0, "balance": -0.00010321}, + { "address": "Qbh6gWsxNtcaKq5sAq4NVkCTXuqyA6pbUm", "assetId": 0, "balance": 0.00003628}, + { "address": "QbJhMqYk94FExie11Vs5y5x7CNUS5e5b5W", "assetId": 0, "balance": -0.00010321}, + { "address": "QbJJho6sTHnqL2ECivtfUrYZTuEgemEha2", "assetId": 0, "balance": 0.00003628}, + { "address": "QbJqEntoBFps7XECQkTDFzXNCdz9R2qmkB", "assetId": 0, "balance": 0.00000103}, + { "address": "QbLPi1Ac6zZGTLURapA6YxyiFYVXH6uZYQ", "assetId": 0, "balance": -0.00010321}, + { "address": "QbmJDoAJ9cjRNM9AuMv5AZc4w83kqosCYS", "assetId": 0, "balance": 0.00000004}, + { "address": "QbMppCoLPnXdzBQBCqXaD1iCBLGKVSW7Z1", "assetId": 0, "balance": -0.00010321}, + { "address": "QbNaKP3udSoKqgRdVR5uik5tb2QrgmyY5w", "assetId": 0, "balance": -0.00010321}, + { "address": "QbnizezPemhpQg1roAMs9MAvJVw4KHiSuq", "assetId": 0, "balance": -0.00010321}, + { "address": "Qbp8CZLnBwphPGwqJGm91LTaFJ4mkXZLgg", "assetId": 0, "balance": -0.00010321}, + { "address": "QbPGzVLU7B9TSrr3fde2u7xoyQfRUpk8st", "assetId": 0, "balance": 0.00003628}, + { "address": "Qbq1ctcvwnkChPmU9PiH4fAExbgTv3fBm3", "assetId": 0, "balance": 0.00003628}, + { "address": "QbQQhQM7XdoGPPjJG1ffNwGEU4tmgUVSyb", "assetId": 0, "balance": 0.00003628}, + { "address": "QbqRyFw7Xu6Nsb4FraaUSe7nUPukuUpekG", "assetId": 0, "balance": 0.00000011}, + { "address": "QbSLbuAxRMqe9vdQpmhbannkJcieXgPfnY", "assetId": 0, "balance": 0.00003628}, + { "address": "Qbtut4Z8a37Mokd5uvsA54WXfBHN1Ho1Kx", "assetId": 0, "balance": -0.00006713}, + { "address": "QbUPbjTu3NpEZQcJp4JcTarLRon4oTiSqi", "assetId": 0, "balance": 0.00003628}, + { "address": "QbUSTCbTkdKgaMJuKiEfJfa32cXTy4sHkr", "assetId": 0, "balance": -0.00010321}, + { "address": "QbvxC3ENqomXp11833APchdjeyCNd49nLj", "assetId": 0, "balance": 0.00000011}, + { "address": "QbXjh5buBXW68AmJBUW2URV4YnM59vMhkP", "assetId": 0, "balance": 0.00003628}, + { "address": "QbxJvwrEHZs7MDE8rbqBwZAZkcywue5F3W", "assetId": 0, "balance": 0.00003628}, + { "address": "QbY4qRTuHee7gX93n7RJytxNXJhMeQcdCP", "assetId": 0, "balance": -0.00010321}, + { "address": "QbYaYDYjTDohUtsbALeR4PQPUXL2qYe3hh", "assetId": 0, "balance": 0.00003628}, + { "address": "QbYTowTHCr9WzfrR6b8uDfJKwL41nG1vyr", "assetId": 0, "balance": 0.00003628}, + { "address": "QbYVbsJ99wWEDNn7fGNgUYuSN1fk6y3T1x", "assetId": 0, "balance": 0.00003628}, + { "address": "QbyVFRE1zKKcprNvpCBx1VfEh9uosYZojs", "assetId": 0, "balance": 0.00003628}, + { "address": "QbzERVYhUEJKvWRVpEeiacV9HcNxjoCzA4", "assetId": 0, "balance": 0.00000011}, + { "address": "Qc1wMMJbivCnM4QjvgJDWqmSQYUfuswhts", "assetId": 0, "balance": 0.00003628}, + { "address": "Qc54dt4km6NrxBMvtEX51jKiuNmHmzEuee", "assetId": 0, "balance": 0.00003628}, + { "address": "Qc9dZchoYfc1eRJhSLXR9rxSHcqNB47Dex", "assetId": 0, "balance": 0.00000026}, + { "address": "QcCBVfL35rxSyQ416L2MBz14FYbNrbeNPx", "assetId": 0, "balance": 0.00003628}, + { "address": "QcCL2sk1nLLE99HgqjGpqLQbCwPtSozBx5", "assetId": 0, "balance": -0.00010321}, + { "address": "Qce2Djqrk2WzG1QhMZ3BqFok9HGsz4wtM3", "assetId": 0, "balance": 0.00003628}, + { "address": "QcEpMZ9NUkLcEv2aWw6FPu9f58CSKVSH8N", "assetId": 0, "balance": 0.00000011}, + { "address": "Qcf5wVLGjgdt57kYsn1D5H6TuWorqb6hww", "assetId": 0, "balance": -0.00010321}, + { "address": "QcfhhsQG9vgVdQULu8RrXaXooJUec1xMj1", "assetId": 0, "balance": -0.00010321}, + { "address": "QcFZ9yCvGESF7gi12jt9XF8RY423c6RfLm", "assetId": 0, "balance": -0.00010321}, + { "address": "QcGdwubfgY14EPjc7peuWn7vz8tKZbWQw4", "assetId": 0, "balance": 0.00003628}, + { "address": "QcGFjReZ7yjNJaCMF1SfXbdrPCGeZhdgCv", "assetId": 0, "balance": 0.00003628}, + { "address": "QcgxbHijQFUaBBD1Xv6k2Fjh2hRPp2AUFg", "assetId": 0, "balance": 0.00003628}, + { "address": "QcHF9YogbuzZhG4fK4116pgE2qrmbkGh2n", "assetId": 0, "balance": 0.00000004}, + { "address": "QcHF9YogbuzZhG4fK4116pgE2qrmbkGh2n", "assetId": 2, "balance": 0.00000004}, + { "address": "QcJwVCyzraPy51uB4xd4f94n2UFYAsznGC", "assetId": 0, "balance": 0.00003628}, + { "address": "Qck27pE28zWwmMoxa2hypGa9X6rhjBBfmJ", "assetId": 0, "balance": 0.00001276}, + { "address": "QckLxn2NgwZZjV92W8VKnHUWiWUVmQrhiJ", "assetId": 0, "balance": -0.00001256}, + { "address": "QcNmqT8CZ6zSZwuRm5LahRZnuGBJRnPY8o", "assetId": 0, "balance": 0.00000011}, + { "address": "QcPPNyDKGk5vQfPEpXQQKvYeidvBZ9T7nS", "assetId": 0, "balance": 0.00003628}, + { "address": "QcPro2T97Q8cAfcVM4Pn4fv71Za4T6oeFD", "assetId": 0, "balance": 0.00000103}, + { "address": "QcQPQmeU1hw8fWQmsBCKEuxg3kRizaQYUz", "assetId": 0, "balance": 0.00003628}, + { "address": "QcqTj1unHj5FXExKu7RpRJHPBMPjGtmXtJ", "assetId": 0, "balance": 0.00003628}, + { "address": "QcrnYL6yNwHKuEzYLXQ8LewG3m2B5k9K5f", "assetId": 0, "balance": 0.00000019}, + { "address": "QcrnYL6yNwHKuEzYLXQ8LewG3m2B5k9K5f", "assetId": 2, "balance": 0.00000015}, + { "address": "QcRYGiF4ffxMUq3CGNrcFP646KbeCcnK66", "assetId": 0, "balance": 0.00003628}, + { "address": "QcU4VhU9ohDXU4k4AUMapgJRYSzEpizjLN", "assetId": 0, "balance": 0.00000011}, + { "address": "QcUA6GT9FiPBbeE7ttBXu1avBHZzDsZg2o", "assetId": 0, "balance": 0.00003628}, + { "address": "QcwEgPdsvF1TugqnHvT2bwXLCLzEKMVk3A", "assetId": 0, "balance": 0.00003628}, + { "address": "Qcx4PE9bn3qXn88XhpDmNSBGS32SmDE8Ds", "assetId": 0, "balance": 0.00002352}, + { "address": "QcXuXDcqzq8goJAqwambRU2Uk9RQ513mV9", "assetId": 0, "balance": -0.00003608}, + { "address": "QcyBacxzvdMP5votSnAJyA39fu9BgYhWmG", "assetId": 0, "balance": 0.00003628}, + { "address": "Qd1Px9vhWuEmF2SbLx3Ez7HhGtifGMa8TJ", "assetId": 0, "balance": 0.00003628}, + { "address": "Qd1Xw41BzN1CgASqsh2PcrrkTKyDs2MVYF", "assetId": 0, "balance": 0.00000011}, + { "address": "Qd1Xw41BzN1CgASqsh2PcrrkTKyDs2MVYF", "assetId": 2, "balance": 0.00000011}, + { "address": "Qd33zAmKqm89UWMes6bfRMMoSNjasehzzX", "assetId": 0, "balance": -0.00010321}, + { "address": "Qd3bVidnA4fhKv1xwHcKsDZC3MUBFhkrUa", "assetId": 0, "balance": 0.00003628}, + { "address": "Qd453ewoyESrEgUab6dTFe2pufWkD94Tsm", "assetId": 0, "balance": 0.00000103}, + { "address": "Qd75TjafsrikBgnfA6Hb6Y9wk45LwiavB1", "assetId": 0, "balance": -0.00010321}, + { "address": "Qd7rmD8PZvKKyJLr4qgvFQzeLPRhYkKcya", "assetId": 0, "balance": -0.00001256}, + { "address": "Qdb27GFXfyWFDKY3urtrKrQRshkeL8hWgt", "assetId": 0, "balance": 0.00003628}, + { "address": "QdFZk74skMUu4rKMPEmcSVwR87LNDe6o3Y", "assetId": 0, "balance": 0.00003628}, + { "address": "QdGbhtkFHUqd9nK9UegxxGXD1eSRYSoKjt", "assetId": 0, "balance": 0.00000011}, + { "address": "QdgbtYSRsDgKZ2PZMKCfoNWfGuvDm2idmP", "assetId": 0, "balance": -0.00010321}, + { "address": "QdHc49iRMiCaanZfD8kGiTZaZxJneDTU7j", "assetId": 0, "balance": -0.00010321}, + { "address": "QdkheUawBwuvhD5J5N21uqypH1hZw1enGE", "assetId": 0, "balance": 0.00003628}, + { "address": "QdkTGqDYde3Y9Q6EgxmBrJGAK2jm4HXspX", "assetId": 0, "balance": 0.00003628}, + { "address": "QdmFGD2ef9gdkUbXpNBuKVkbGGnBRBoReS", "assetId": 0, "balance": -0.00010321}, + { "address": "QdP6twdTsJpq3eLDgi83t6LH367gauJLqo", "assetId": 0, "balance": -0.00010321}, + { "address": "QdRzjsQrz8edqeRNX7VcASbSz2hPfvX783", "assetId": 0, "balance": -0.00010321}, + { "address": "QdsakiJEhKaKGtG4ue2k5xJdt6kYsxwPba", "assetId": 0, "balance": 0.00003628}, + { "address": "QdsiBFfTQUPMrS6NuWdcYCU94t8M64EcPf", "assetId": 0, "balance": -0.00010321}, + { "address": "QdsMQUyuWyYT5Sit8YSMW9bKjBhfq8MwRY", "assetId": 0, "balance": -0.00010321}, + { "address": "QdSQjxAdRwfg4JgdaXNp5CZwTFL8ARwDJf", "assetId": 0, "balance": -0.00010321}, + { "address": "QdtAQm1EGNgM7QDSaC2qvV9WdpRHwpApUT", "assetId": 0, "balance": 0.00000011}, + { "address": "QdTY1v63aMSibfPV2sJTAJZu2mqDP4dMZV", "assetId": 0, "balance": -0.00010321}, + { "address": "QdudYG9SDw5WYzfoj9oq3QC4abHx8ZWCce", "assetId": 0, "balance": 0.00003628}, + { "address": "QdwSxr3t4hdGHjQFy6EVGR9yGMipefsTuo", "assetId": 0, "balance": 0.00003628}, + { "address": "QdxDaHTEX5cUg4S6ohMAi6mE8wQZCqQBoT", "assetId": 0, "balance": 0.00003628}, + { "address": "QdXdUxnyKGGo7eEfTcx85oEikNe5nYnuwa", "assetId": 0, "balance": 0.00003628}, + { "address": "QdXe21sjY8smjVmiAUgZY8xWVzwgxMgK5A", "assetId": 0, "balance": 0.00000011}, + { "address": "QdXe21sjY8smjVmiAUgZY8xWVzwgxMgK5A", "assetId": 2, "balance": 0.00000011}, + { "address": "QdXMNPHhbt2kiwkp7NPskBsrZudxZ6gXN2", "assetId": 0, "balance": 0.00003628}, + { "address": "QdyzBSPBNLfyCfdPNBn86EcXUZeDdkCYLm", "assetId": 0, "balance": 0.00096104}, + { "address": "QdZ5Krd84CX6oh6jorEhYyL7zdCacrPAj2", "assetId": 0, "balance": 0.00003628}, + { "address": "Qe29bjnmk29z19Nw3xBkbWMqMzy7SkzZ57", "assetId": 0, "balance": 0.00003628}, + { "address": "Qe7RxFfsV5JNkQNuK9UVvtTQMXhcCcTTTf", "assetId": 0, "balance": -0.00010321}, + { "address": "Qe9S8zA27FPdPVVLcVQj9noaKsuwySPKdq", "assetId": 0, "balance": -0.00010321}, + { "address": "Qe9VPzQp3h4Kg3DHSHBUQ3AM3AiRBfCDfX", "assetId": 0, "balance": 0.00000019}, + { "address": "Qe9VPzQp3h4Kg3DHSHBUQ3AM3AiRBfCDfX", "assetId": 2, "balance": 0.00000015}, + { "address": "QeAa7yawpJqQYk7PNisVD89HezskBRecH6", "assetId": 0, "balance": 0.00003628}, + { "address": "QeaDGU85fpffwsw9ngmd98QsT6NaFyFFed", "assetId": 0, "balance": 0.00000011}, + { "address": "QeaDGU85fpffwsw9ngmd98QsT6NaFyFFed", "assetId": 2, "balance": 0.00000011}, + { "address": "QeAHiq28seiCm7wxMoo4NWJtAoBVMtZrpc", "assetId": 0, "balance": 0.00003628}, + { "address": "QeaJHCCTy7AebbPeF1scsBLbezcBHAKtKt", "assetId": 0, "balance": 0.00001276}, + { "address": "QeAwxFMkYMmTJN5dysZtYAaq2SAxjeYrL4", "assetId": 0, "balance": 0.00001276}, + { "address": "Qec83tt9eX6Ng41GE8PU91GWMi72Hk74K5", "assetId": 0, "balance": 0.00003628}, + { "address": "QeCTKHwG4zypj1bV7uNAzyMc4hwed2EFga", "assetId": 0, "balance": 0.00001276}, + { "address": "QeH2ajmr3ca3t2g6xcnbmFeGYd9BeACvA8", "assetId": 0, "balance": 0.00001276}, + { "address": "QekAWuiw9PUQfygRRF31aLemzHBLkRWpiU", "assetId": 0, "balance": -0.00010321}, + { "address": "QepkE9dJdWsYZZdYRP5bV4NbzQnpABWR4m", "assetId": 0, "balance": -0.00010321}, + { "address": "Qeq85FoJpxtzoDM93WiNQQCXiuiFynRQzm", "assetId": 0, "balance": 0.00000011}, + { "address": "Qeq85FoJpxtzoDM93WiNQQCXiuiFynRQzm", "assetId": 2, "balance": 0.00000011}, + { "address": "QeSh3t1AnaRcRThkkUTvvdMEouixCADeVh", "assetId": 0, "balance": 0.00000033}, + { "address": "QeSh3t1AnaRcRThkkUTvvdMEouixCADeVh", "assetId": 2, "balance": 0.00000033}, + { "address": "QesUoX7rrugxqGFCk4AYntQkoxvXcLpEoS", "assetId": 0, "balance": 0.00003628}, + { "address": "QeSzLpUw9as4LUHTJ3CNK6SW9okCkU1qMG", "assetId": 0, "balance": 0.00003628}, + { "address": "QeTgFSAQj6AihCoJ5cfNJt2ZCDkSUGSAnB", "assetId": 0, "balance": -0.00010321}, + { "address": "QeU4z63x84mZJmwjxZLKWkJbRu46iP1H2z", "assetId": 0, "balance": -0.00010321}, + { "address": "QeUE2MQKGopfmrcLknKfrDnJ8ddoktxrHr", "assetId": 0, "balance": -0.00010321}, + { "address": "QeVGUYQSiVgAq4mZ2KQswbkkxBxxH3jb9Q", "assetId": 0, "balance": 0.00003628}, + { "address": "QewrEYLQ7anM7UyPvLEEitQpfD4pjt1pQQ", "assetId": 0, "balance": -0.00010321}, + { "address": "QeY5cbodjaunb2anyhQMwurmZtZUuuDCc1", "assetId": 0, "balance": 0.00003628}, + { "address": "QeZm1XBbyycGh8mdcoBTGpD2Z4v5unA2yt", "assetId": 0, "balance": 0.00003628}, + { "address": "QezTNFB9czsSYhbJN9YMLNrhRNtmazJrfC", "assetId": 0, "balance": 0.00003628}, + { "address": "Qf2eC5PFMzfqdcS5xq4vfE4n8Hmt3iuYuk", "assetId": 0, "balance": -0.00010321}, + { "address": "Qf3gdKYQqKgXs6sAVkyW2uHkautxNbzgvJ", "assetId": 0, "balance": 0.00001276}, + { "address": "Qf4QdaFNUN7m3eKJfst1vzq87n3cgER6gT", "assetId": 0, "balance": -0.00010321}, + { "address": "Qf9AMghvwAMwoRi38YzDNNxQHrEwAwgyNo", "assetId": 0, "balance": 0.00003628}, + { "address": "QfbX8JJupEw5ckNtU4upQgET35oLTr5e6v", "assetId": 0, "balance": 0.00003628}, + { "address": "Qfd3gxberH9K6ipiV33VH3TUooNFYyV1iu", "assetId": 0, "balance": 0.00003628}, + { "address": "QfdMLMLZC9Kt15DRAdxwNfaYdGBEMF9Sb4", "assetId": 0, "balance": 0.00003628}, + { "address": "Qffd1qyWXmC5ZgUc4GQzPCHkeH29DS52H4", "assetId": 0, "balance": 0.00003628}, + { "address": "QfhJM5CpX5MhfjCcopwfw7pgS6w1hVJ49E", "assetId": 0, "balance": 0.00000037}, + { "address": "QfhJM5CpX5MhfjCcopwfw7pgS6w1hVJ49E", "assetId": 2, "balance": 0.00000037}, + { "address": "QfjL32jLsxtumbfx6ufmfCFCBccVCQFkrh", "assetId": 0, "balance": 0.00000011}, + { "address": "QfJVbN5dRnMUSedZH68HhYCTJzjbUo121Q", "assetId": 0, "balance": 0.00003628}, + { "address": "QfKsnQFFJWMkXEz2bdSuB734uMNpi5VAaD", "assetId": 0, "balance": -0.00010321}, + { "address": "QfmM8dgfikTB2FYVuJ9owzQXVm8wP7T4QT", "assetId": 0, "balance": 0.00000014}, + { "address": "QfnbnWrRQ4HNDQvtg3wG2B1eC4ycUsFqZz", "assetId": 0, "balance": 0.00003628}, + { "address": "QfPcwetW3BErP4ySTurxFJSHpNkNXPEhGk", "assetId": 0, "balance": 0.00000011}, + { "address": "QfPcwetW3BErP4ySTurxFJSHpNkNXPEhGk", "assetId": 2, "balance": 0.00000011}, + { "address": "QfpUpwgV5h6SQqaywGvxvBzzV9D774993x", "assetId": 0, "balance": 0.00003628}, + { "address": "Qfr6suiRoJVGWgmxrAb5sdZVWWuPm1aXqD", "assetId": 0, "balance": -0.00010321}, + { "address": "QfRykgxR139CmZ4nDjmFQvaSmNiv576ZYT", "assetId": 0, "balance": 0.00003628}, + { "address": "Qft1Ckorj3uckvuTjokt6k2FB15oNkVW5a", "assetId": 0, "balance": 0.00001276}, + { "address": "Qft1ktvJ14eBFjpJaphT24ks4WRcN3K6tB", "assetId": 0, "balance": 0.00000022}, + { "address": "Qft1ktvJ14eBFjpJaphT24ks4WRcN3K6tB", "assetId": 2, "balance": 0.00000022}, + { "address": "QfTxdUv4M5LWuaoybQwh1VZ8843Wqq1r1t", "assetId": 0, "balance": 0.00003628}, + { "address": "QfUu2KAEuoxBKHMNFMKaryyoX7vRTSvCFP", "assetId": 0, "balance": 0.00003628}, + { "address": "Qfw8iJLRok3vFrenNxt2o4DatGY3hThsmr", "assetId": 0, "balance": -0.00010321}, + { "address": "QfWuDf4QE2ygs3mT1nokqqgQgFfgYm2BMo", "assetId": 0, "balance": -0.00010321}, + { "address": "Qfxkjavp3UR7tuG988Hau1PF3Um27fU6VX", "assetId": 0, "balance": 0.00003628}, + { "address": "Qg1yzP82SghJT7o3kgcWMMF6UYYb6BS8c1", "assetId": 0, "balance": 0.00003628}, + { "address": "Qg2273uygJvXdtp2kmjWgPEy48nX56ctZj", "assetId": 0, "balance": 0.00003628}, + { "address": "Qg32vHmYCUq16co8mj4Ljb8bWzWu2eWmyF", "assetId": 0, "balance": -0.00010321}, + { "address": "Qg6B3mCqHBUY6jm6fL2ddtUvWTZea3xcfV", "assetId": 0, "balance": -0.00003608}, + { "address": "Qg7TPhUD5sns2pJiLjxnktRAx71WXsNp9y", "assetId": 0, "balance": 0.00003628}, + { "address": "Qg9d4zDLvzhqqjDvRz99jiFoRnygTyzeZg", "assetId": 0, "balance": 0.00003628}, + { "address": "Qga1LWYfbXkBaNm5LCh4HwSW8L3g8MFzSm", "assetId": 0, "balance": 0.00003628}, + { "address": "QgBfw49fpZzCL7FRLwMsq8677ffZLk1XBa", "assetId": 0, "balance": 0.00003628}, + { "address": "QgcF6KgVZ9eDAMHJdSEeAtp91t931VKZMv", "assetId": 0, "balance": 0.00003628}, + { "address": "QgCQq4cFaGrJhwvKs4XwvccKiLZ8GVMCXR", "assetId": 0, "balance": 0.00000011}, + { "address": "QgECFJiiri2dDN4zA32URvbdDid2cFrJwM", "assetId": 0, "balance": 0.00000051}, + { "address": "QgECFJiiri2dDN4zA32URvbdDid2cFrJwM", "assetId": 2, "balance": 0.00000011}, + { "address": "QgEGaSaoCj1bGxyj35qaZcpb23Px2bBJmq", "assetId": 0, "balance": -0.00010321}, + { "address": "QgesuKa3zwx8VAseF1oHZAFHMf29k8ergq", "assetId": 0, "balance": 0.00003628}, + { "address": "Qgfh143pRJyxpS92JoazjXNMH1uZueQBZ2", "assetId": 0, "balance": 0.00000103}, + { "address": "QgFjfyApbjupAa1PLBdy5NGNZWXEha1p9T", "assetId": 0, "balance": -0.00010321}, + { "address": "QggJ9Rnh99rRJMjsp6nLSuR4m8FAve8Wfe", "assetId": 0, "balance": -0.00010321}, + { "address": "QgHhee3CSRavmr7h87XSsLb3esiQUyRjxj", "assetId": 0, "balance": -0.00010321}, + { "address": "QgHRqFHrhTDL4TmnzXpQDtno4q24Q24uL9", "assetId": 0, "balance": 0.00003628}, + { "address": "QgHsU3UbVH2HWd3cZKsivtCTMcZjsyEYjc", "assetId": 0, "balance": -0.00010321}, + { "address": "QgJdTosTZQPzBYiWQSeCDw5zGWxa96zfkA", "assetId": 0, "balance": 0.00003628}, + { "address": "QgkGF35JZnfzzzZ3GcrLjdiE7DWGzsoLGz", "assetId": 0, "balance": -0.00010321}, + { "address": "QgmEtScSZWJmTUAidCZKj6gDr3LznZ6rr4", "assetId": 0, "balance": 0.00003628}, + { "address": "QgmJAg1X3MaQ1kp8ABKe7j6okY3RdumNfE", "assetId": 0, "balance": -0.00003608}, + { "address": "QgNtUZEQAbDAn2zbBVu8KLexZHLvDN3Rcw", "assetId": 0, "balance": 0.00003628}, + { "address": "QgnZT74VfseKiSPZgzMBVE7JpRs3N2dMs2", "assetId": 0, "balance": 0.00003628}, + { "address": "Qgp16aMcdiS2EUkxCm5NSZgB8DixGK51zT", "assetId": 0, "balance": 0.00003628}, + { "address": "QgpXUK9QEyJgFedP68iSPqD91CwoRnpB6X", "assetId": 0, "balance": 0.00003628}, + { "address": "QgqM5bKs3tNqKNAnVeaQp4oaYMXCmX6YJr", "assetId": 0, "balance": 0.00000011}, + { "address": "QgRfZM6pz7JoX8N3YheCqZCLkZbLZmAzKQ", "assetId": 0, "balance": 0.00003628}, + { "address": "QgSCgLQWuMCd9867ygUToidDaHstCaaK7X", "assetId": 0, "balance": 0.00003628}, + { "address": "QgsifXqfJtdsNbxqGYh3hExEpWWZDg9rKK", "assetId": 0, "balance": -0.00010321}, + { "address": "QgsxkxTBhBwtccex56cwbaYydp3imnikJe", "assetId": 0, "balance": 0.00003628}, + { "address": "QguDWvRKfdRv1bHDV5wqqnY1drJQTn6365", "assetId": 0, "balance": 0.00003628}, + { "address": "QgUhCiEHoA8ERQFzog8ubuCd321f1VYbDP", "assetId": 0, "balance": -0.00010321}, + { "address": "QgVRwXN3x9suBrh7Dc1HPnRiMeuLqZ5FJk", "assetId": 0, "balance": 0.00003628}, + { "address": "QgVZb632eqF1eLQm9gBGuBtyp9Dyz2FKUK", "assetId": 0, "balance": 0.00000011}, + { "address": "QgYQpgDWSMi6Rma7VqzYsuG7TWq1ChSxEv", "assetId": 0, "balance": -0.00010321}, + { "address": "QgZAyh4znJgzsb5tKGsYXXKhaZ2zYitqVg", "assetId": 0, "balance": 0.00003628}, + { "address": "Qh2cPc2Bn8fceg3wCCvkSk38oEkQ6KxWaG", "assetId": 0, "balance": -0.00010321}, + { "address": "Qh4EmrLoRwePL6mi9XZ85s1d2pkkfzj3RV", "assetId": 0, "balance": -0.00010321}, + { "address": "Qh55kmNyvRAmA7KZPAiZ5gmJSLGNNAxL5m", "assetId": 0, "balance": -0.00010321}, + { "address": "QhCcvRt4jmyFtjeqeHGeU4Z1DKdRFGmxs3", "assetId": 0, "balance": 0.00003628}, + { "address": "QhCyt8DqiumxXXJka9ErkieUWGW5AA8SvD", "assetId": 0, "balance": -0.00010321}, + { "address": "QhD2RCxdxXRKku893rvdtJbnv1bt2QR5TD", "assetId": 0, "balance": -0.00010321}, + { "address": "QhdoF3Kt3dV5DkuPgTmvH3RNzgZrSK9o6W", "assetId": 0, "balance": 0.00002352}, + { "address": "QhDPwTaDTzjSCwYHZvphyqRVhHYZ9CWmzz", "assetId": 0, "balance": 0.00003628}, + { "address": "QheP4ZKWsYzu14fewMdUayA4rkXpuH8a4p", "assetId": 0, "balance": 0.00003628}, + { "address": "QhErhzUapqPRDdXYyaS9nG8cvbCibhRUpq", "assetId": 0, "balance": -0.00010321}, + { "address": "QhH8txpLcffTmttkYjS9AWqi2Vwz6hHCBp", "assetId": 0, "balance": 0.00003628}, + { "address": "QhjbKpRtrTHegRr28KHVgP6XAZyJyjkapw", "assetId": 0, "balance": 0.00003628}, + { "address": "QhkyUQcRTxSDzTzUGJoPGnLC9N1ZiDcwnt", "assetId": 0, "balance": 0.00003628}, + { "address": "QhM6LS7TCiAiWMbvXWMWSDNJVEwHPdLb94", "assetId": 0, "balance": 0.00003628}, + { "address": "QhPwExMZk8mW4FvH2HtQGbq5mU2sTHNS9B", "assetId": 0, "balance": 0.00003628}, + { "address": "QhPXuFFRa9s91Q2qYKpSN5LVCUTqYkgRLz", "assetId": 0, "balance": 0.00003628}, + { "address": "QhQdzLn36SDgrgoMfvdZAkoWtTUHpB3acJ", "assetId": 0, "balance": 0.00003628}, + { "address": "QhqoEihESYJnVDTBWpEPsXub6c7eCJgAma", "assetId": 0, "balance": 0.00003628}, + { "address": "QhQsFX4iYf9f5zQp5CLQPQVzSEX2fTcSbx", "assetId": 0, "balance": -0.00010321}, + { "address": "QhsactZ9HZTkUSff3fWpRNxSZjueunYiF1", "assetId": 0, "balance": 0.00003628}, + { "address": "QhspjBT3mpnao5EeLqEY3HJFXv42uPpCks", "assetId": 0, "balance": 0.00003628}, + { "address": "QhTHNmR4YgYJu5o2uzGkgUHFpg1pYu9KU3", "assetId": 0, "balance": 0.00003628}, + { "address": "Qhuos9t2XkBCmiFiroQFwQ7CaULAZ9YBnj", "assetId": 0, "balance": 0.00003628}, + { "address": "QhwZ6thwxwaucJfbNxB2LoA17GZqfaA1D7", "assetId": 0, "balance": -0.00010321}, + { "address": "QhYD58sT9N8b6jhNgcmVnLXRuVCnbnuxUd", "assetId": 0, "balance": 0.00001276}, + { "address": "QhYS1Ag1RjYVUSGYYKyvQXSRWN9nyFGnnS", "assetId": 0, "balance": 0.00003628}, + { "address": "QhZJFejortmvM99apbw83n9RVFhFpNUCLF", "assetId": 0, "balance": 0.00003628}, + { "address": "QhzyudB9g5TieSbCi2SBtm9sS8ia2hq4oe", "assetId": 0, "balance": 0.00003628}, + { "address": "Qi1W3hiPZWH6wfGt2imicU7upZcqHy7RBv", "assetId": 0, "balance": 0.00001276}, + { "address": "Qi2Cw4zZFvHLQnZVhwjM1ygqbn6nDEB4ZN", "assetId": 0, "balance": 0.00003628}, + { "address": "Qi3N6fNRrs15EHmkxYyWHyh4z3Dp2rVU2i", "assetId": 0, "balance": 0.00000096}, + { "address": "Qi73xBLZg3PMJELtLd9GebkDCW7Kjk1juc", "assetId": 0, "balance": 0.00002352}, + { "address": "Qi8EEW1qUuG63yRShzKBh1Wb7r88UeCNZZ", "assetId": 0, "balance": -0.00001256}, + { "address": "Qi8j8NLi2wBg7JUAb9qwctXwbyLbmbN6pp", "assetId": 0, "balance": 0.00003628}, + { "address": "QiA5SLN3NASu4EStTmCAzvR4ih1Z1TwhwC", "assetId": 0, "balance": -0.00010321}, + { "address": "QibuD4c6gvXgS4iut7q3sXuVb23rgFJq2M", "assetId": 0, "balance": 0.00003628}, + { "address": "QiBYApdEYRwsFYjt59UJqZV55wcwykvhsh", "assetId": 0, "balance": 0.00000015}, + { "address": "QiBYApdEYRwsFYjt59UJqZV55wcwykvhsh", "assetId": 2, "balance": 0.00000004}, + { "address": "QicRwDhfk8M2CGNvpMEmYzQEjESvF7WrFY", "assetId": 0, "balance": 0.00000040}, + { "address": "QiDji6mSFEF3PjGnKfZvMwJrR1GQtnf6Pd", "assetId": 0, "balance": 0.00003628}, + { "address": "QiERDpXv985tgbL39GsKrrkbrfmKBj6bpN", "assetId": 0, "balance": 0.00003628}, + { "address": "QiFUUj4GvfHTTuAhFseuoWZm3wYemqxSDn", "assetId": 0, "balance": 0.00003628}, + { "address": "QiGkwhUZJRsg1AzcQofw78KVmbGeoobTyf", "assetId": 0, "balance": 0.00003628}, + { "address": "QiGN3Kce81GdoiWkztj58hypZ1qBUiMPnZ", "assetId": 0, "balance": 0.00003628}, + { "address": "QiGumRw6CnrTkWKkAp7pXYSBvtNsDPaoGH", "assetId": 0, "balance": 0.00003628}, + { "address": "QihzNPXWQC5HqjmzqT91GzhNpVXmveGJq6", "assetId": 0, "balance": -0.00010321}, + { "address": "QiJQdet8ziyDeCijhJXFE7MnWbX5XQpn2T", "assetId": 0, "balance": 0.00003628}, + { "address": "QiQUssukhoo1ft4G9Mxa8JpViqFW4PdBjJ", "assetId": 0, "balance": 0.00000011}, + { "address": "QiRDWHPRQcp6jQrtjqNYRDtkvGnsjryXF5", "assetId": 0, "balance": 0.00003628}, + { "address": "Qis1kSR77JUx957Lw4oV1kZsHAtZ6bL52W", "assetId": 0, "balance": 0.00003628}, + { "address": "QiSRUnc4FbFX4Mb4b8i2Aa9ebXb6r3qhNr", "assetId": 0, "balance": 0.00002352}, + { "address": "QisSQZ7Et7Rfzx2SCC2o9UDSeRZWMyFKWc", "assetId": 0, "balance": 0.00003628}, + { "address": "Qit8cfptq7zJXj9xiZRGoT8Lz36TeLjcsS", "assetId": 0, "balance": -0.00006713}, + { "address": "QiYuVgjzYfEwobgrTFVsSa2R3ut4tL2rm1", "assetId": 0, "balance": -0.00010321}, + { "address": "Qj2USguA2xGYbUFHwU9jzJwmyGCiTaQEcS", "assetId": 0, "balance": 0.00001276}, + { "address": "Qj6NdK4qoLrsHkWoDNhasSkrLLsudFMDWp", "assetId": 0, "balance": -0.00010321}, + { "address": "Qj6nk7RCJcB6gB5SZYGnKrqk6umyVD2XWT", "assetId": 0, "balance": 0.00000028}, + { "address": "QjBMFFPhQryK31Uk7jzhLaC4grbq4Lv3XM", "assetId": 0, "balance": 0.00003628}, + { "address": "QjdMUSewptx5M9KXUrx8HPSVZPXqa8JDVC", "assetId": 0, "balance": -0.00010321}, + { "address": "QjEaMxcBKMsj91ytKe6GdTBJP8Mu1Ru3r4", "assetId": 0, "balance": 0.00003628}, + { "address": "QjEAs2or122weKppv5zALzoQzXxbsDjy3f", "assetId": 0, "balance": 0.00003628}, + { "address": "Qjf1sJh7Y7e16QWjExgzQt7o2Mqxrgw977", "assetId": 0, "balance": -0.00010321}, + { "address": "Qjfqk23ZTP9wNLTPLyQzewhxgmjxh8cwv7", "assetId": 0, "balance": 0.00003628}, + { "address": "QjgGeEkyiXa43pyqkXxZbvAChQpVYfUyKz", "assetId": 0, "balance": 0.00003628}, + { "address": "QjKRbeTYYi53Pu4Ph3t8seavsoay3N8zpi", "assetId": 0, "balance": 0.00003628}, + { "address": "QjKzZNNuBYWvZkGP1YtTnbPXY12DPmhWcP", "assetId": 0, "balance": -0.00010321}, + { "address": "QjLMLprtMqwfKEN3XoocfDRVSWkbKQwm7d", "assetId": 0, "balance": 0.00000011}, + { "address": "QjMWr9osCo2eJVZyzRn5zNURn6azCC4Agx", "assetId": 0, "balance": 0.00003628}, + { "address": "QjQosFc13zX2kN52Miyo3DuBYU288jNkdW", "assetId": 0, "balance": 0.00003628}, + { "address": "QjrC8NXwR8gFkEauvRwCPxqHroPFqAJbhK", "assetId": 0, "balance": 0.00000011}, + { "address": "QjrC8NXwR8gFkEauvRwCPxqHroPFqAJbhK", "assetId": 2, "balance": 0.00000011}, + { "address": "QjrCFCi6dqvka4UELg2SHhM2oWnQWepd1o", "assetId": 0, "balance": 0.00003628}, + { "address": "QjTRoy39Bfq1DJD6UPiHvcCVgrA663WkSf", "assetId": 0, "balance": 0.00003628}, + { "address": "QjVm1ZaT62korzr9XvRxmJppyFF4sdafeT", "assetId": 0, "balance": 0.00003628}, + { "address": "QLcnnJygHWRvCYyxk1EUwMMWpJicgp8WkF", "assetId": 0, "balance": 0.00003628}, + { "address": "QLdw5uabviLJgRGkRiydAFmAtZzxHfNXSs", "assetId": 0, "balance": 0.00003628}, + { "address": "QLfUgN4QRkHacD6PdxfUUQUBG7NXkqz2Pg", "assetId": 0, "balance": 0.00003628}, + { "address": "QLhxpFjnYi8HToiHep6X3okP1U45bpz54S", "assetId": 0, "balance": 0.00003628}, + { "address": "QLi8RY1wquju2jXpEgJ1f9e2i1NyBQhxJy", "assetId": 0, "balance": 0.00001276}, + { "address": "QLj3L7YAX1TqCBvkMcXJ7pKoVyVwhjhhMA", "assetId": 0, "balance": 0.00002352}, + { "address": "QLjE5xQbBcALTSpnuu3Ey5SG7jqj4ke8hZ", "assetId": 0, "balance": 0.00003628}, + { "address": "QLk4souHeUSaT5jKezcmKtjUexZKyqXjqb", "assetId": 0, "balance": 0.00003628}, + { "address": "QLqSs841jDXCJQ1RJ4xq68V5V2FQtP9GkU", "assetId": 0, "balance": -0.00010321}, + { "address": "QLr2d2rVviocEfqta6cRb9uKZfH8YEGb2P", "assetId": 0, "balance": -0.00010321}, + { "address": "QLu1ZhFYAHdq5YPemwBBy6wNtJS9ZnsiV4", "assetId": 0, "balance": 0.00003628}, + { "address": "QLwMaXmDDUvh7aN5MdpY28rqTKE8U1Cepc", "assetId": 0, "balance": 0.00000103}, + { "address": "QLxSSDU5QNfpSzWpBuLSauuj6uEecqUcD1", "assetId": 0, "balance": 0.00003628}, + { "address": "QM1jwcy9hgFmjkbNHcJTv9ksS6q3gpeHNd", "assetId": 0, "balance": -0.00010321}, + { "address": "QM7gZsy5p5qxuyikPEqMUcUArDHtJ2A1Kv", "assetId": 0, "balance": -0.00010321}, + { "address": "QM9zVbXXnfrtQ1X7zPQ5zxPYPAWTaVMXqZ", "assetId": 0, "balance": 0.00000036}, + { "address": "QM9zVbXXnfrtQ1X7zPQ5zxPYPAWTaVMXqZ", "assetId": 2, "balance": 0.00000022}, + { "address": "QMC3FQSa83vDUH3ApyoVnJe6P8JTdedCiR", "assetId": 0, "balance": -0.00010321}, + { "address": "QMEFWMLuPfmFzzGH5WCECz4VE6HjenLic9", "assetId": 0, "balance": 0.00003628}, + { "address": "QMFG3ucD5qJXShA9uzD3gVnhfKfTnNnJpX", "assetId": 0, "balance": 0.00003628}, + { "address": "QMfWg9oJg49izXMeRWrsErgNnBD6mJcKiX", "assetId": 0, "balance": 0.00003628}, + { "address": "QMGgED2eawpZZNoRTGghwkMP9NLUoCYoVw", "assetId": 0, "balance": 0.00003628}, + { "address": "QMH8qGMEHvw9YPUCE2fX9recgGfSWh6Aao", "assetId": 0, "balance": -0.00010321}, + { "address": "QMHbcfCReXRZPd1xaRaFMdFdKXXUsLC3nc", "assetId": 0, "balance": 0.00003628}, + { "address": "QMiGaZbcWXdj61Rn1UGVAPgg8s31puuX1v", "assetId": 0, "balance": 0.00003628}, + { "address": "QMJwdufHY9dMoARHCUyGbMPAqUB4BcqGKm", "assetId": 0, "balance": 0.00003628}, + { "address": "QMMh94Pfs5LVE4xJee1yggViqP1YDdQHT4", "assetId": 0, "balance": 0.00000011}, + { "address": "QMNfNWsJuFwiSufnVwWpGU7bqcNgdTKF7o", "assetId": 0, "balance": 0.00003628}, + { "address": "QMozpRT9aUunfmPh7EtQ6LPoth2JFJWBXC", "assetId": 0, "balance": 0.00003628}, + { "address": "QMpb5Gxr9PTReeCN6r3BZgPMXozMpmmaQM", "assetId": 0, "balance": 0.00003628}, + { "address": "QMPNM6p7zADqo2hp4DTXdPEZLgKTQ7qJJr", "assetId": 0, "balance": -0.00010321}, + { "address": "QMQKzygMix6WVy2J1kdepSSHjJnk2nK6MK", "assetId": 0, "balance": 0.00003628}, + { "address": "QMSfC5v8AF5SntsKuKnsgLt2EbaMNWvNhz", "assetId": 0, "balance": -0.00001256}, + { "address": "QMsKXQAYKmR7dBH4P3kMLiKzYatK3h1CeS", "assetId": 0, "balance": 0.00000011}, + { "address": "QMt6UeGcA8BLkLkfAjy8AShnBtu8ECgooc", "assetId": 0, "balance": -0.00010321}, + { "address": "QMtm8wVPHGE3qHg2hMaj6SZ78D5eXw3VWZ", "assetId": 0, "balance": 0.00003628}, + { "address": "QMuWNAJ2tbeViHtBUN3yD2KARrrzcanLAd", "assetId": 0, "balance": 0.00003628}, + { "address": "QMyV3ZofyJrRTmZfpoHrPoNZo7oA9vnoZY", "assetId": 0, "balance": -0.00010321}, + { "address": "QMz6NGa9TZzChT1sMRXjiM7uR5q6Nn34Fb", "assetId": 0, "balance": 0.00003628}, + { "address": "QMzbEtBrrjBDoFFz6n7bd6uzLWy3iXeZhN", "assetId": 0, "balance": 0.00003628}, + { "address": "QMZJe6ZxJ7rVQmX2nUqH6JdUAXXyJTvJHu", "assetId": 0, "balance": 0.00003628}, + { "address": "QN289qYSyeBD5jS3vKnqHZ4nNnpqbeh2n6", "assetId": 0, "balance": 0.00001276}, + { "address": "QN8RijNFo7SDDKYgF5yiWuh86UhWpcpdGL", "assetId": 0, "balance": 0.00003628}, + { "address": "QN95wtwKG2yT7NZkpU1q1QmFpNSYcdQVZL", "assetId": 0, "balance": 0.00003628}, + { "address": "QN9E7MDY914bqaw13TxLrs47iPzW9rJF8e", "assetId": 0, "balance": -0.00010321}, + { "address": "QNAVKy8r1WXTSisFcdPjwwYHsbMW479MwX", "assetId": 0, "balance": 0.00003628}, + { "address": "QNb5Za9VKLMni3MzyzuxxjSAykXbPGSVU1", "assetId": 0, "balance": 0.00003628}, + { "address": "QNCHqRw177Ct9ExD7FiAJaN6w6yhqkNuiY", "assetId": 0, "balance": 0.00003628}, + { "address": "QNEF6iVnzXPAqhJf4x46DhXSnRrPjgqWiC", "assetId": 0, "balance": 0.00003628}, + { "address": "QNfi5TZ8LYdK1acZz2VKnChvhY5t7QRWe1", "assetId": 0, "balance": 0.00003628}, + { "address": "QNfmBzXcb3gLXmBteM4oToqakRrCFXjVuS", "assetId": 0, "balance": 0.00003628}, + { "address": "QNFsnWD53JWrQJzVAj4JWr75qS3Pu4MMXk", "assetId": 0, "balance": -0.00010321}, + { "address": "QNHBFkVUopmpYyguufnSe6DUcbThtEUu47", "assetId": 0, "balance": -0.00010321}, + { "address": "QNHdGeFJmPcDdN8prPzPL4bk2dpnJ2ZZFr", "assetId": 0, "balance": 0.00003628}, + { "address": "QNiTnonHpXTeUrgNdyYWVDPP4ZdjkLpW72", "assetId": 0, "balance": 0.00003628}, + { "address": "QNiuxkAEGm1a5K3EzPhEvnFHVCQQWuoEoh", "assetId": 0, "balance": -0.00010321}, + { "address": "QNMDKE7XTujNQkuQorcHXw6hL7qRvyaTjr", "assetId": 0, "balance": 0.00000011}, + { "address": "QNMZnDAbYDzKLYRVNpJFcEnr3411rQBguw", "assetId": 0, "balance": -0.00010321}, + { "address": "QNoRAk6XmihoFnqP6SCEFNhR66n3McCaTF", "assetId": 0, "balance": -0.00006713}, + { "address": "QNoxXt2xDKrM51adtcFLcW92wk615qA64H", "assetId": 0, "balance": 0.00003628}, + { "address": "QNqBFbZXg5STiW6F4Lj1a1v8dMaFiACpJR", "assetId": 0, "balance": 0.00003628}, + { "address": "QNrYqn8pMjx8ax7jBeQa6onzrjEEapCABf", "assetId": 0, "balance": 0.00003628}, + { "address": "QNsiHhrAQUDk5h3ecLw8bAiF2179aggSsK", "assetId": 0, "balance": 0.00003628}, + { "address": "QNuSfDpB84q9Xydrpk7Rhu5mNY5BfWSVcc", "assetId": 0, "balance": -0.00010321}, + { "address": "QNVKrjEq5bZdiDtgo64m5kz87rTHqCwvCP", "assetId": 0, "balance": 0.00000011}, + { "address": "QNw21XRyVhudVTc15XcZZ7giKGWVAndSig", "assetId": 0, "balance": 0.00003628}, + { "address": "QNw9xAm9TUerin9QsapCPL9mV6zmoXyJrh", "assetId": 0, "balance": 0.00003628}, + { "address": "QNwVgYAQZxc6KD9FMUT2dmQBLaXdnxv7yF", "assetId": 0, "balance": 0.00003628}, + { "address": "QNXCHtt6hwppn3DjKVHEn2ybmPehgNGuV8", "assetId": 0, "balance": 0.00003628}, + { "address": "QNzMU13YxVuueojNCXegU3cDXUfju7TLkB", "assetId": 0, "balance": -0.00010321}, + { "address": "QP2hds2BNPhsK7fyMHgGApzQDuGbjhEwbD", "assetId": 0, "balance": 0.00003628}, + { "address": "QP3J3GHgjqP69neTAprpYe4co33eKQiQpS", "assetId": 0, "balance": 0.00000099}, + { "address": "QP5fR7t4SJE6F5U2q2gPLEh9U63GCEr4pB", "assetId": 0, "balance": 0.00003628}, + { "address": "QP7yGYN9fJufzuFWRSjwUXhdnVokypQ3Es", "assetId": 0, "balance": -0.00010321}, + { "address": "QP8rHEbqCr8D1aUHEn3rKa2Jgtahcjf6We", "assetId": 0, "balance": -0.00006713}, + { "address": "QP8xG56L8b28h1mguSk9LuzNhxbHgAoL9b", "assetId": 0, "balance": 0.00000011}, + { "address": "QP91M2haBGwDcayzvGp7wBBM1pugC5Sse1", "assetId": 0, "balance": -0.00010321}, + { "address": "QP9vU5yTsBjuTSFxH5Cb9VXYNRHKhMNAJ4", "assetId": 0, "balance": 0.00003628}, + { "address": "QP9y1CmZRyw3oKNSTTm7Q74FxWp2mEHc26", "assetId": 0, "balance": 0.00003628}, + { "address": "QPcTWoAhYWmwjmWbQAS8muisrQVaLJMbg7", "assetId": 0, "balance": 0.00000011}, + { "address": "QPcTWoAhYWmwjmWbQAS8muisrQVaLJMbg7", "assetId": 2, "balance": 0.00000011}, + { "address": "QPEbvVBWDG7qgy4smY8nWiie78Vec8qiT9", "assetId": 0, "balance": 0.00000011}, + { "address": "QPEd3HwgZ9w6W9eYnEMuS4NmjD8iR3DMQM", "assetId": 0, "balance": 0.00003628}, + { "address": "QPfP9syFgebRP5A1s2DK7kC1L6hWFoLjoB", "assetId": 0, "balance": -0.00010321}, + { "address": "QPGvKDAhG86Z9UDyo6pSvDLUQkSCi44JfT", "assetId": 0, "balance": 0.00003628}, + { "address": "QPLoqpwAoytvpQKwvJ6GRsaRcVZ3xnYgVB", "assetId": 0, "balance": 0.00003628}, + { "address": "QPmNcXdK2EVmxZ2KSeGS5N6s8Koikwnutc", "assetId": 0, "balance": 0.00003628}, + { "address": "QPnfdCQNDDP4LTpUQPEiydgmp734mXNvb5", "assetId": 0, "balance": -0.00010321}, + { "address": "QPNuXTLsQaBzUTENu4mmhLyqTAKAVfdVym", "assetId": 0, "balance": 0.00003628}, + { "address": "QPpr2JS24d9maQtXLpNQuLqivWe17VNth8", "assetId": 0, "balance": 0.00003628}, + { "address": "QPQaXAcVz6jtP9X5oCwUhHrC6PY7jpof7r", "assetId": 0, "balance": 0.00003628}, + { "address": "QPqfuZpmyA6cK6WUFwcGeKH2Te1aegkHBM", "assetId": 0, "balance": 0.00003628}, + { "address": "QPqXoYhTPiDdSuwcAj9JvrnBuzTYDBJEmv", "assetId": 0, "balance": -0.00010321}, + { "address": "QPREQjU2defiYdgA33HDiLNGBpxtuebeqE", "assetId": 0, "balance": 0.00000011}, + { "address": "QPrh9z8gNmRe5SU2zmBHSbZzXawkHDiDwy", "assetId": 0, "balance": 0.00003628}, + { "address": "QPRJwmpAh2A9ed1V4ib2GYat4XEZTebPqr", "assetId": 0, "balance": -0.00010321}, + { "address": "QPsjHoKhugEADrtSQP5xjFgsaQPn9WmE3Y", "assetId": 0, "balance": 0.00000082}, + { "address": "QPUMyJ59kkrp75tDzDPxSyw1GWCrbC2cS2", "assetId": 0, "balance": 0.00000004}, + { "address": "QPusqAVBVFGAAeE7RdospttA18AuyLP7sB", "assetId": 0, "balance": 0.00003628}, + { "address": "QPV6pAxUghP23w3KDEv3PDcD9EvAypdvbJ", "assetId": 0, "balance": -0.00010321}, + { "address": "QPVCG6EUkxcuznnDRf4aDLUNSUTnWiKeA7", "assetId": 0, "balance": 0.00003628}, + { "address": "QPwESkkM7hCQ8gSa3cgY3sXB27cQMMrkU9", "assetId": 0, "balance": -0.00010321}, + { "address": "QPWxc25kgMu2ZsFnZwGz8yXdSbNnmgig6s", "assetId": 0, "balance": 0.00003628}, + { "address": "QPWzseBZj9UDGTASKeub5QTGwhpvTvGrAf", "assetId": 0, "balance": 0.00003628}, + { "address": "QPYBoSu8KdPoNGkpjZo7FNy6br2Etzx7q9", "assetId": 0, "balance": 0.00003628}, + { "address": "QPYfRd1uhnAgqkZNmjNCjgPhkguMnHWuc4", "assetId": 0, "balance": 0.00000022}, + { "address": "QPYfRd1uhnAgqkZNmjNCjgPhkguMnHWuc4", "assetId": 2, "balance": 0.00000011}, + { "address": "QPyx2bNiAnJEjitfeAh8jZXzQVKio2B7Mi", "assetId": 0, "balance": 0.00003628}, + { "address": "QPZJVL5PD7ZDEWa2TfN6nx8h55MraPk6SR", "assetId": 0, "balance": 0.00003628}, + { "address": "QQ5qnof5pUgJem8NPsAPgYdENL88cNqSj9", "assetId": 0, "balance": 0.00003628}, + { "address": "QQ6FA4TgpqPc8kN4Sp9LVUZ7Wcix4kT3rc", "assetId": 0, "balance": 0.00003628}, + { "address": "QQ9VZH256J59hAQaHvbqB2DJDPfkvo8R2U", "assetId": 0, "balance": -0.00010321}, + { "address": "QQa3MTgdnru5B7wSqPcq7qXcZcpbDQ7oyE", "assetId": 0, "balance": 0.00000004}, + { "address": "QQaKBSjAt9RK2bqJoSriR77X4ULstGzrFQ", "assetId": 0, "balance": 0.00003628}, + { "address": "QQAuaqYCU2XfTuCkNn4KPbNA7txNN2om62", "assetId": 0, "balance": 0.00003628}, + { "address": "QQbzLNiPHMqtrjGYuHXNgED4F6Pc89t7am", "assetId": 0, "balance": 0.00002352}, + { "address": "QQEWYGZBbmdLL4HrQrAtnyCdzsm67GxAhr", "assetId": 0, "balance": 0.00000011}, + { "address": "QQEZEGWt3sAPwEWYD2RQ6tMwnpkayG81dY", "assetId": 0, "balance": 0.00000011}, + { "address": "QQFabMW4DtU23uUhZRe47Q4F4h2uTHvgcq", "assetId": 0, "balance": -0.00010321}, + { "address": "QQfd4mHdR3YvUXgtq1t6s5RxbnVdagqLiY", "assetId": 0, "balance": 0.00003628}, + { "address": "QQfwxmBGXXU6U88DeYqpp9k3j99g5deGD4", "assetId": 0, "balance": 0.00003628}, + { "address": "QQjiCpwLkxEdvYa5EQvrKoxAL6dA6uJCq7", "assetId": 0, "balance": 0.00003628}, + { "address": "QQoENGwx2bpj24aF9cuGUFd7GVWPH8Led3", "assetId": 0, "balance": 0.00003628}, + { "address": "QQPYyoE3Bm2vh8Wr5aaBNyirC8dd3BhBGH", "assetId": 0, "balance": 0.00000011}, + { "address": "QQrnqFh6AedkwRSAEzWWJUfLVtJPbfNurK", "assetId": 0, "balance": 0.00003628}, + { "address": "QQRwKAbAtVFVbydiwAFmoUivVMPxrND78o", "assetId": 0, "balance": 0.00003628}, + { "address": "QQuhcRELLCkgcc8UTGXKLQfMGY5RWMKwf3", "assetId": 0, "balance": 0.00003628}, + { "address": "QQXgH4CnQCB76BbXhsApu6ShhohFfvoXv7", "assetId": 0, "balance": 0.00003628}, + { "address": "QQYp1TiGWbRChHY8fWzeNSYrBSbyczwkcK", "assetId": 0, "balance": 0.00003628}, + { "address": "QQzMut6erjgSKCpZ1dHDcjKcj9KAce7cug", "assetId": 0, "balance": 0.00003628}, + { "address": "QR5xsQro2R42oU1bcXoZoqxqBsaKvoZkPG", "assetId": 0, "balance": 0.00003628}, + { "address": "QR9UR5QUE7yAwPyios25WQdworma6k8iLf", "assetId": 0, "balance": -0.00006713}, + { "address": "QRaDef6H2zYfefqLwYGmUg7T6DAqo6DDqc", "assetId": 0, "balance": 0.00003628}, + { "address": "QRCkZ5zUcgo7mMthsYznkbjxRgeqyKDKtD", "assetId": 0, "balance": -0.00010321}, + { "address": "QRCtc67FTNKS5zVXM8omw8F55h9DP7herL", "assetId": 0, "balance": 0.00000011}, + { "address": "QRctujZqsh51nbvfxmJzXcCoJDA5hRxdmp", "assetId": 0, "balance": 0.00003628}, + { "address": "QRcvSzSNezuGkLqAoeAHqVvQDGUP6CTKeq", "assetId": 0, "balance": 0.00003628}, + { "address": "QREtYDhP4HkpeCCZroemuGXMGVFoZHH3Lp", "assetId": 0, "balance": 0.00003643}, + { "address": "QREtYDhP4HkpeCCZroemuGXMGVFoZHH3Lp", "assetId": 2, "balance": 0.00000015}, + { "address": "QRFHr4jnVgvAsPTubeSrh8bPy1yzwzYaWD", "assetId": 0, "balance": 0.00000011}, + { "address": "QRHCneJApSW2qe2uuo1QkFq5Xb4qx5YfpK", "assetId": 0, "balance": -0.00010321}, + { "address": "QRHj3FnBzzDM314JVi4HNcvAHit5EXamLo", "assetId": 0, "balance": 0.00002352}, + { "address": "QRiAaFKLPgScKPUjGHEA5uTa1gjt8ZRXSv", "assetId": 0, "balance": -0.00010321}, + { "address": "QRJRPTC431ortbmXizawh3JM64Vd1qGWhu", "assetId": 0, "balance": 0.00003628}, + { "address": "QRk5TG57SQGLkybXUqxBnobADTFGj9GR3Z", "assetId": 0, "balance": 0.00003628}, + { "address": "QRKRk5HVADsN1LHygK7q2pA7dWnYKnPpCT", "assetId": 0, "balance": 0.00000011}, + { "address": "QRmdkrmtJrjXyrGLGEPB981KLw5GddvXgw", "assetId": 0, "balance": 0.00001276}, + { "address": "QRMYnBhWD1ncLWiMrMeRiMpcTnc4dv3sTb", "assetId": 0, "balance": -0.00010321}, + { "address": "QRnLRt2D4hkKFsyxq2UUfUH5mGwchJc25h", "assetId": 0, "balance": 0.00003628}, + { "address": "QRP5BTeMLUqWbkES3gRqFHsWh4ey8Bot2v", "assetId": 0, "balance": 0.00003628}, + { "address": "QRQa5tidBjxWd29s8Rqvmcv7Lm1irtPnio", "assetId": 0, "balance": -0.00010321}, + { "address": "QRrvsUPv9Xv6EL9M3uEWiJiSMDV4uQc1zv", "assetId": 0, "balance": -0.00010321}, + { "address": "QRt11DVBnLaSDxr2KHvx92LdPrjhbhJtkj", "assetId": 0, "balance": 0.00000103}, + { "address": "QRtRELSSASzqiYy2FtNcrePH6TVnqJkv9B", "assetId": 0, "balance": 0.00000011}, + { "address": "QRtwMe8xmGic45KkXJ2mADFmbLq4fnnY4g", "assetId": 0, "balance": -0.00010321}, + { "address": "QRu2rd4V1wnQ8yifhk18JgCTyvpoZMTg8j", "assetId": 0, "balance": 0.00003628}, + { "address": "QRUbzEbLd7fRjAx2fBdXAH4QS1WQyetvDc", "assetId": 0, "balance": 0.00003628}, + { "address": "QRViAhwyGycgNbRZ4ywQHEWtVDcN6L1e5q", "assetId": 0, "balance": -0.00010321}, + { "address": "QRWEbcRnLoGccAndtLcGgpeQFH2ZBcMqHo", "assetId": 0, "balance": 0.00003628}, + { "address": "QRWEbzH4niUcu9dL3Yq42X4j89aqQk3qWw", "assetId": 0, "balance": 0.00003628}, + { "address": "QRwxvvk5UBLjwYQTXxZG1Xa8yYssGKTUKj", "assetId": 0, "balance": 0.00003628}, + { "address": "QRZWZQP7Tmi4orAWcHWhXpfmjtK4TdCuSu", "assetId": 0, "balance": -0.00010321}, + { "address": "QS1i9K7iJb49TA4w43VSC3fEURF6bRXvw9", "assetId": 0, "balance": 0.00003628}, + { "address": "QS2wQpMWBXaN8tV1hvWoJVSXtFKoJ4jBDJ", "assetId": 0, "balance": -0.00010321}, + { "address": "QS3kGsoFhyPeeyCbQcGiMH3LvP2KYNaKxe", "assetId": 0, "balance": 0.00003628}, + { "address": "QS49qPLS7w4xqBdcbYqwUwKjY2wN7AVmxu", "assetId": 0, "balance": -0.00010321}, + { "address": "QS7K9EsSDeCdb7T8kzZaFNGLomNmmET2Dr", "assetId": 0, "balance": 0.00003628}, + { "address": "QSApBAn8pD6jmLs6j4WwxeCa341Crb9yp4", "assetId": 0, "balance": 0.00003628}, + { "address": "QSAtkxer4LwkdQqzStB82K74CNSXuamx8x", "assetId": 0, "balance": 0.00003628}, + { "address": "QSbHwxaBh5P7wXDurk2KCb8d1sCVN4JpMf", "assetId": 0, "balance": 0.00000103}, + { "address": "QSBVtqNoaM9pfi8zMgMY9pTQhaF6XjMUZU", "assetId": 0, "balance": -0.00010321}, + { "address": "QScBgSw74MquesXmVJxerX3YgyhtShRr4q", "assetId": 0, "balance": 0.00003628}, + { "address": "QSE7sy84kVtiB5tQiRWcSXZmQX5NG3tM1k", "assetId": 0, "balance": 0.00003628}, + { "address": "QSFhD2auWxoBqBzMZggf1FqTzoUxz7cddo", "assetId": 0, "balance": 0.00003628}, + { "address": "QSGB4Rd2xhd6UmA9LALTQ4f89Tfsz5VajU", "assetId": 0, "balance": 0.00000011}, + { "address": "QSgFEURyKLVh8z84Wxb6MJxDSvY7DaRuUM", "assetId": 0, "balance": -0.00010321}, + { "address": "QSH7dFCpRkbxvfrAeAxK81u5HyBbgbUHs9", "assetId": 0, "balance": 0.00003628}, + { "address": "QSHLf7MR3LtKN5oeWewqJPEmgMBDVRB6Pb", "assetId": 0, "balance": 0.00003628}, + { "address": "QSJmwFYNx8mGn1n791WFzJW8BqJqZZZwRt", "assetId": 0, "balance": 0.00003628}, + { "address": "QSjqYaBA8euuKZsvJQu9moQmaPkPgjnxUL", "assetId": 0, "balance": -0.00010321}, + { "address": "QSKaxQHPYatp6YcE33CmtwiovP1qZAJZSe", "assetId": 0, "balance": 0.00003628}, + { "address": "QSkicapNH35a3UebSxxSMCfntBhwwi6veW", "assetId": 0, "balance": 0.00003628}, + { "address": "QSMZpdZWbMZQa7wxcywzrzaWTQTN216mjk", "assetId": 0, "balance": 0.00003628}, + { "address": "QSq7VrrognfVmGLrPhmpRVZZmHw5LApnD5", "assetId": 0, "balance": -0.00010321}, + { "address": "QSq8y4ZrSbF55ZddWNcw1ett2LDtjQEvNn", "assetId": 0, "balance": 0.00003628}, + { "address": "QSU5p6Q78dQM44hCDCJwnWgbVyXPg9wMH4", "assetId": 0, "balance": 0.00003628}, + { "address": "QSUruudcrhmPuM9v4JAoSnAdeQpFjQUtwG", "assetId": 0, "balance": -0.00010321}, + { "address": "QSwwCXx9hJgM7ZAVvFb1oQjrMcSfgBcDqy", "assetId": 0, "balance": -0.00010321}, + { "address": "QSZSkfeNcaK2fKLJiF6TwVuZuEt4opALN4", "assetId": 0, "balance": 0.00003628}, + { "address": "QT2X4TSA8mG7UitNFwY5DkkV7WS1RRPn7R", "assetId": 0, "balance": 0.00003628}, + { "address": "QT5PXfTynkrhckdqd6L5NyGxeWVBgXtMQC", "assetId": 0, "balance": 0.00002352}, + { "address": "QT6K1KJ3ED3wm7Fdc2ETW27spHWdjYhAXG", "assetId": 0, "balance": -0.00010321}, + { "address": "QTBG5F778g7j2yw82ReDZuAqyLC3xe1RCu", "assetId": 0, "balance": 0.00003628}, + { "address": "QTbrzCB9nnAv7Vno5dEAw4NXxAfVoNwyWA", "assetId": 0, "balance": 0.00003628}, + { "address": "QTd6P8ZuoG36VRE9W3VhtvRWQgHN3qTkhT", "assetId": 0, "balance": -0.00010321}, + { "address": "QTdSGHWUaEjx1kW1AZdRAZPCkaNwcqDCPe", "assetId": 0, "balance": 0.00001276}, + { "address": "QTdWujAFt2ErKotw4cjiorqLUqCiWCMz9i", "assetId": 0, "balance": -0.00010321}, + { "address": "QTecbuir4YPxLQ9c9Ht1TVrkHTKfnAALBd", "assetId": 0, "balance": 0.00003628}, + { "address": "QTEE4ZJXv68ke4841HWjTLAAU8mfccxwbE", "assetId": 0, "balance": 0.00000004}, + { "address": "QTFg4go5uJ1oZidqRCXqu7miyUKiqzWuD2", "assetId": 0, "balance": 0.00169901}, + { "address": "QTGBUbMv9cKMxbrrCQBiXtj6XUEyYumNns", "assetId": 0, "balance": -0.00010321}, + { "address": "QTGeQqn3XEFdnnCqvifCFXYdKym7SaHzTd", "assetId": 0, "balance": 0.00000011}, + { "address": "QTgtYSdWErieArhJ7eznKSv451TqCxMYxa", "assetId": 0, "balance": 0.00003628}, + { "address": "QTKKxJXRWWqNNTgaMmvw22Jb3F5ttriSah", "assetId": 0, "balance": 0.00000103}, + { "address": "QTM9jb15o2kU9fT6ARQdYJGWfH5xC1vtCt", "assetId": 0, "balance": -0.00010321}, + { "address": "QTMczbPVBQ4Yvr3GdjS6YeLjRCBn8hvx68", "assetId": 0, "balance": 0.00002352}, + { "address": "QTMTFswUU83XVmk6T4Gez7qUJCccbAad7S", "assetId": 0, "balance": 0.00000004}, + { "address": "QTpYQqRyMekaEuECziirzy3HvCVofZS1wJ", "assetId": 0, "balance": 0.00003628}, + { "address": "QTRAvkMBrHEt4sDYAa6dHUNeGjmcfAYtys", "assetId": 0, "balance": 0.00003628}, + { "address": "QTrJucEocNy7MHXvVqWWN82TGnm7ssue2H", "assetId": 0, "balance": -0.00010321}, + { "address": "QTSi7WDsgJtCmGpE9vJot32dmozM21bDrR", "assetId": 0, "balance": 0.00003628}, + { "address": "QTSrDNbWFxFUzpCX9MnoGuBKgwP1oQqjsg", "assetId": 0, "balance": -0.00010321}, + { "address": "QTsutcJRvjMV8MhTvuetGL6rPEAnvcdYZB", "assetId": 0, "balance": 0.00003628}, + { "address": "QTTrv8SWR8huV8TFYUEQhfZ1j1JmtL5p8G", "assetId": 0, "balance": 0.00000011}, + { "address": "QTTrv8SWR8huV8TFYUEQhfZ1j1JmtL5p8G", "assetId": 2, "balance": 0.00000011}, + { "address": "QTtXS6fZGThRLq4qgkwM4ngBYkLoFyZ3bK", "assetId": 0, "balance": 0.00003628}, + { "address": "QTVLRwoopfg7qSG9CMfCPJz3UydnT3jDxD", "assetId": 0, "balance": 0.00000014}, + { "address": "QTW9TTM7fM4ghv1UAfa4L6w25D9PsKeh3f", "assetId": 0, "balance": 0.00001276}, + { "address": "QTWc2J5jHUEeqyxSCQvnZu1GXuEWdFN8U3", "assetId": 0, "balance": -0.00010321}, + { "address": "QTx98gU8ErigXkViWkvRH5JfaMpk8b3bHe", "assetId": 0, "balance": 0.00003628}, + { "address": "QTxK2iBYyj3Qwcfwo8vjtVvmLQmZVVME1D", "assetId": 0, "balance": 0.00003628}, + { "address": "QTy8J1dtbWc5KFBYJfLcoQAbzktV4JsxNp", "assetId": 0, "balance": 0.00003628}, + { "address": "QTyokTJrR4b2y76An3BFUEbqQy5vvg76iN", "assetId": 0, "balance": 0.00003628}, + { "address": "QTZEiy2RgGgyPpkMWE6trKYRHSGqPMufM5", "assetId": 0, "balance": 0.00003628}, + { "address": "QTZh1TbhwyYWUdsMaTLVnWikotRBDvRwVz", "assetId": 0, "balance": 0.00003628}, + { "address": "QU5sn5xsq5CwtzSQ8bqbgQkFR4CVUymtA5", "assetId": 0, "balance": 0.00003628}, + { "address": "QU8vFmk1xUoRTFQuRupck4HSeeuYFAVMjw", "assetId": 0, "balance": 0.00003628}, + { "address": "QUaH6kB6Jk5mZfsFpdyKvYzFA12j4g2Bss", "assetId": 0, "balance": -0.00010321}, + { "address": "QUarYYMPRodjEEBKrGsTsufPa1pc5M9mVk", "assetId": 0, "balance": 0.00003628}, + { "address": "QUAXAog3G9F8cJMC3KL4iGGFC3AK5hrzzP", "assetId": 0, "balance": 0.00003628}, + { "address": "QUCbBSPjDjygRJehHwjcXtM7PngUXKMiLW", "assetId": 0, "balance": 0.00003628}, + { "address": "QUCbFnjNwYfugM29oh6syCMnpr68vXuQjN", "assetId": 0, "balance": -0.00010321}, + { "address": "QUdjqijDoyc83K4WcMW1sCn7zLd2t1WTqn", "assetId": 0, "balance": 0.00003628}, + { "address": "QUGo9SErgc6ceB5aBzcSJDNqBkQ9eaCKZS", "assetId": 0, "balance": 0.00003628}, + { "address": "QUgUMBsdauY9p7ahjkEkxPnH51vqZ6fEit", "assetId": 0, "balance": -0.00010321}, + { "address": "QUhQUjdExXnbX6BYNSHNYohv8WUQgDpCYP", "assetId": 0, "balance": 0.00003628}, + { "address": "QUjcZYLfVsmJdB57w4rbvPKuSe26hoX8nA", "assetId": 0, "balance": -0.00010321}, + { "address": "QUjm9fPRs4wbvXmwUYdMDg3HdNxGuR1DBo", "assetId": 0, "balance": 0.00003628}, + { "address": "QUJyCt8ZMDauaH4avg84gCbLY5Es2KJVFM", "assetId": 0, "balance": 0.00003628}, + { "address": "QUKKwug9PNai3DBggXUXP8Ag7WmR5SVUR4", "assetId": 0, "balance": 0.00003628}, + { "address": "QUNYcKorTAjcFEFH2kLuGzTHDSXHbTm9n4", "assetId": 0, "balance": 0.00000011}, + { "address": "QUNYcKorTAjcFEFH2kLuGzTHDSXHbTm9n4", "assetId": 2, "balance": 0.00000011}, + { "address": "QUo3i1Ae9apv8muRUZuKaz7oTbRdKDWKgd", "assetId": 0, "balance": 0.00003628}, + { "address": "QUon9BuHPfvwS74tju9apvSioPGRh2R9f2", "assetId": 0, "balance": 0.00003628}, + { "address": "QUQdkr2SVCBcDTXVseG7MuZshQxSwyGZB2", "assetId": 0, "balance": 0.00003628}, + { "address": "QURLLepAaEaUQuQKT3PQ1zMMTD4w8ztuxD", "assetId": 0, "balance": 0.00003628}, + { "address": "QUt4pPZnFH3Sd1NhQNg5CEbKhGZHceTqNb", "assetId": 0, "balance": 0.00003628}, + { "address": "QUTM1cfWdFFehQx2MdNENSqZKh1aqR4Z7K", "assetId": 0, "balance": 0.00003628}, + { "address": "QUvoLFfkuVuRe1KGMLQS4nUHry6CBTuTYz", "assetId": 0, "balance": 0.00000004}, + { "address": "QUvtYEENi8wPXqCE2kereZaNxNgXrVivYr", "assetId": 0, "balance": 0.00003628}, + { "address": "QUw11tpoaCGqYvXdNoLE67vbTaRGvkAP8i", "assetId": 0, "balance": 0.00003628}, + { "address": "QUwdTXDoZ5BPMeW53e2epqV987jWej2Nk6", "assetId": 0, "balance": -0.00010321}, + { "address": "QUXga5K8nzd9EqYtvEesZWEYuA688h6D3d", "assetId": 0, "balance": 0.00003628}, + { "address": "QUxh6PNsKhwJ12qGaM3AC1xZjwxy4hk1RG", "assetId": 0, "balance": 0.00000059}, + { "address": "QUXqBSukt3Lmp8qBdCMtaM2P4qFGTBarCw", "assetId": 0, "balance": 0.00003628}, + { "address": "QUxxuGuZX141B6ZzDds6oojPHGqEM3cPNV", "assetId": 0, "balance": 0.00000011}, + { "address": "QUYF2HhJF1tF4avi5xByPwwWhguHYXXLWL", "assetId": 0, "balance": -0.00010321}, + { "address": "QUYfELYYXqHxEELSfDixQUL6ZqxvgqCtxE", "assetId": 0, "balance": -0.00001256}, + { "address": "QUZCxNDBcv74PfrP9dXk1SbEsaQnKdb2Nd", "assetId": 0, "balance": 0.00003628}, + { "address": "QUZQPWhrxpze32vGiux6wa85kg9iwuhCDx", "assetId": 0, "balance": 0.00003628}, + { "address": "QUzUCfoakDqBaL5zBgfvTKLHcuxbUfB38Q", "assetId": 0, "balance": 0.00000011}, + { "address": "QUzUCfoakDqBaL5zBgfvTKLHcuxbUfB38Q", "assetId": 2, "balance": 0.00000011}, + { "address": "QV2HChYd7opM1r6oYaX7KA5VUoKdiUuagg", "assetId": 0, "balance": 0.00000007}, + { "address": "QV4496JU7VU9hwfZBwwprEGUv2d1RedQWz", "assetId": 0, "balance": 0.00003628}, + { "address": "QV7bS2gnJnTzL38eD2YjNvBZFwQwe4Mw5U", "assetId": 0, "balance": 0.00003628}, + { "address": "QVA3VtN9yYQDuptoTRCXoPDtvuwgW4pjH6", "assetId": 0, "balance": -0.00001256}, + { "address": "QVaeUJbQHTamCcbULtSiiMFHsM2fqQunsy", "assetId": 0, "balance": -0.00010321}, + { "address": "QVbDaDCrHEq8s5nfhHhFf3m712kAqbzFL5", "assetId": 0, "balance": -0.00010321}, + { "address": "QVbSXYN5wdKL5u5QZnJiYQgY9BeTGmfs7z", "assetId": 0, "balance": 0.00003628}, + { "address": "QVBWwms8Goc9ruUSPFtBt48b36uzQmKVdo", "assetId": 0, "balance": 0.00003628}, + { "address": "QVchuzuoTKxNi2aMhype6sS7HCRLhdDrvw", "assetId": 0, "balance": 0.00001276}, + { "address": "QVeSskDtxCQz7xj5GQcHrPgK5Kdtevjgc4", "assetId": 0, "balance": -0.00010321}, + { "address": "QVgFYrrQV9feh45kAT6DyBonxdiJxvpmzh", "assetId": 0, "balance": -0.00010321}, + { "address": "QVHWbjbpjg9zPXfyET7Sjb7JA9BBMWL9Qr", "assetId": 0, "balance": 0.00003628}, + { "address": "QVi5jjTjJNoUg9kXSKAQPzxNA3yYsKBnEE", "assetId": 0, "balance": 0.00003628}, + { "address": "QViKVZa3M3ar7RBRSBMTx8FdzLh1zxUhN8", "assetId": 0, "balance": 0.00003628}, + { "address": "QViPTQGYNRXN7SQQEoNKvFnEW56X2sBqj8", "assetId": 0, "balance": -0.00006713}, + { "address": "QVLuvt9krmxXwQPAeAhxzhuMF5i8F4aNs8", "assetId": 0, "balance": 0.00000015}, + { "address": "QVLuvt9krmxXwQPAeAhxzhuMF5i8F4aNs8", "assetId": 2, "balance": 0.00000015}, + { "address": "QVnHHnf5ZPBpbLZQabtjZBzi9TPgtqABqc", "assetId": 0, "balance": 0.00003628}, + { "address": "QVpVKaXziXmP8qawtxqaFN8mHFAqvuiWzY", "assetId": 0, "balance": 0.00003628}, + { "address": "QVrvy4ac2jBTfxyCKB7MLimqJooTDBApmS", "assetId": 0, "balance": 0.00003628}, + { "address": "QVSo56b2nsrEbWzi3FBkGQCLJNyk9b9j7a", "assetId": 0, "balance": -0.00010321}, + { "address": "QVSp1MvSTBut7shWdddfwdsWf9c7snBDwS", "assetId": 0, "balance": 0.00003628}, + { "address": "QVSqUrNFR4mPTMa7UdVmNKZTSaDVAv8XXF", "assetId": 0, "balance": 0.00000015}, + { "address": "QVSqUrNFR4mPTMa7UdVmNKZTSaDVAv8XXF", "assetId": 2, "balance": 0.00000015}, + { "address": "QVuksgNt3QAr7KCrkxtE5FWrczfgLKxs4H", "assetId": 0, "balance": 0.00003628}, + { "address": "QVurebcEbe4USR4xcS3Mbk12mhxsjRX31u", "assetId": 0, "balance": 0.00000052}, + { "address": "QVurebcEbe4USR4xcS3Mbk12mhxsjRX31u", "assetId": 2, "balance": 0.00000048}, + { "address": "QW1ip98ypmMmcSRjCRkS7Jd1SfneQbU7fq", "assetId": 0, "balance": 0.00001276}, + { "address": "QWb8NhsKVEnfM8NSMPdSWSdn1T4zkCDDFD", "assetId": 0, "balance": -0.00010321}, + { "address": "QWBFK5h61ZxGfqQpEkwwKTcLAo8t9VWe4K", "assetId": 0, "balance": 0.00003628}, + { "address": "QWBrxCkBSMNaL5ssPEawjfP9qUdurrFmP3", "assetId": 0, "balance": 0.00003628}, + { "address": "QWC7MydcEFhjENmCS2YABKY5F5BQd8XYyA", "assetId": 0, "balance": 0.00003628}, + { "address": "QWcgaTFfxt1cZL7hn9G8ayo81WT13S5ECM", "assetId": 0, "balance": 0.00003628}, + { "address": "QWcqcnuDeNpeUYqcvsJtXYdCpDb35ehAv6", "assetId": 0, "balance": -0.00010321}, + { "address": "QWe1iPDudLU189BggPykbH1DrAeaFEgX6W", "assetId": 0, "balance": 0.00000059}, + { "address": "QWe1iPDudLU189BggPykbH1DrAeaFEgX6W", "assetId": 2, "balance": 0.00000059}, + { "address": "QWFHfqYNCYW9EN63zdprYSFQx2ApS6Hj2z", "assetId": 0, "balance": 0.00003628}, + { "address": "QWigG4GAT8eQ6rmNo4AdcGjF5ygSTAV1Q1", "assetId": 0, "balance": -0.00010321}, + { "address": "QWinRb65f2g3yBoaZvTrQKQk7CW7vfBgGX", "assetId": 0, "balance": 0.00003628}, + { "address": "QWK94e5PzrBN5gHrFd77dHeP5XtCiWxVj5", "assetId": 0, "balance": 0.00001276}, + { "address": "QWKYjxBUt2c6BHm26c4k7U8iF9eUEEAeQy", "assetId": 0, "balance": 0.00003628}, + { "address": "QWL7kZp6Pdd1bhxZ6SXPhVf5g7GParG9CC", "assetId": 0, "balance": 0.00003628}, + { "address": "QWLpsGYrkF2cy3tH6DCxso7kXZpZJvv13e", "assetId": 0, "balance": 0.00000011}, + { "address": "QWMEPx9QfK4ErsHx4RwyoWL1cf4xqpBzXy", "assetId": 0, "balance": -0.00010321}, + { "address": "QWmkcX9Ak4EJMZ6JZskF5uqBxiqK6R9c8s", "assetId": 0, "balance": 0.00003628}, + { "address": "QWN4qgyBfSn9TRTJM9e8ftzuAZmSuadmt5", "assetId": 0, "balance": 0.00002352}, + { "address": "QWomBbcXNTdkyuPFUafwtBfpbxHzmUZzqi", "assetId": 0, "balance": 0.00000011}, + { "address": "QWomBbcXNTdkyuPFUafwtBfpbxHzmUZzqi", "assetId": 2, "balance": 0.00000011}, + { "address": "QWRpDYNycvqQrL9RmMDraL1hjTRBbghekz", "assetId": 0, "balance": 0.00003628}, + { "address": "QWS3EtVcBFz9iFjeANx1hTNN8Pfxi2Ft6H", "assetId": 0, "balance": 0.00003628}, + { "address": "QWuW2YMygVtWieUo6a4yayD1xFDWdnmo5j", "assetId": 0, "balance": 0.00000011}, + { "address": "QWvTdm9LU1GSX9q6Rrvgx7xjo2iuV2Gxn1", "assetId": 0, "balance": 0.00000011}, + { "address": "QWvTdm9LU1GSX9q6Rrvgx7xjo2iuV2Gxn1", "assetId": 2, "balance": 0.00000011}, + { "address": "QWwBj6cFoM5EAE7sXULVw2BjVMCPTxDmVs", "assetId": 0, "balance": 0.00003628}, + { "address": "QWwcZuuDMpZtzFvjQthW6FUaUwpkzsmBN4", "assetId": 0, "balance": 0.00003628}, + { "address": "QWwrtjBL4ah965XPXHYJhymreC9jyryNLZ", "assetId": 0, "balance": 0.00003628}, + { "address": "QWYxUJmR6M6tvyxZASux3pxWuq1iWqTPei", "assetId": 0, "balance": -0.00010321}, + { "address": "QWzjhwJg7u7EAfJVvRffiENs5ufhevZNso", "assetId": 0, "balance": 0.00003628}, + { "address": "QX5g6nyYJrbiMxdcHdcHgbfA8jURQcEZGZ", "assetId": 0, "balance": -0.00010321}, + { "address": "QX6TiCGH3oJKucGW2vEYU3kRKBuXSZLZtn", "assetId": 0, "balance": 0.00003628}, + { "address": "QX8mxo977eANNG6Q59Z4dCW3eX3HPBrZ1R", "assetId": 0, "balance": 0.00000011}, + { "address": "QX92FzQwtqm4svY7TR4gxt1aVAjEzdNnKo", "assetId": 0, "balance": 0.00003628}, + { "address": "QXapyoyeuUZ44m8PdJ2XcMdADkrMeAeRzF", "assetId": 0, "balance": 0.00003628}, + { "address": "QXaXcaaL1eDZQaECk47BCsjHojWGfHLcw2", "assetId": 0, "balance": 0.00003628}, + { "address": "QXB9jbqCrYBA68vgzrr8Z8bqMXrCvyU7Z1", "assetId": 0, "balance": 0.00003628}, + { "address": "QXdyEzgLMniSSzf2PS7hQhqUX6XKQemJnv", "assetId": 0, "balance": 0.00001276}, + { "address": "QXeW3vzV8Rfe9kUbm15BW9dFFuXuBq8feb", "assetId": 0, "balance": 0.00003628}, + { "address": "QXFVYCWTAnM3FVhpYkin3Yu7WPb8w7NNZ2", "assetId": 0, "balance": -0.00010321}, + { "address": "QXHEZ4axuNq91K5wW9zaNSvtLzsdsQ1yVz", "assetId": 0, "balance": 0.00002352}, + { "address": "QXHMjGxDjd1RbNN4o2XdmXBdpiS6emQ8QL", "assetId": 0, "balance": 0.00003628}, + { "address": "QXHmtFXzf4D7PEu73NfBm3sZyeuGrm3QC5", "assetId": 0, "balance": 0.00003628}, + { "address": "QXjqVCQ8RaaC7T6Tyiag26Ruj1Tyrx9PvP", "assetId": 0, "balance": 0.00003628}, + { "address": "QXKmtkHHwaUQzGeHHG2dFiHUnKAp815Mzq", "assetId": 0, "balance": 0.00000026}, + { "address": "QXKmtkHHwaUQzGeHHG2dFiHUnKAp815Mzq", "assetId": 2, "balance": 0.00000015}, + { "address": "QXm5e16Lq6dnYwpZJ8Rn2cME3ziHZfRRnp", "assetId": 0, "balance": 0.00001276}, + { "address": "QXmAdL5wEpgWbTSgnHJgdfQmKkhnx4EfaC", "assetId": 0, "balance": -0.00010321}, + { "address": "QXmjUhJ7hmcQRrZ1UvnJAeFhp3aYiwL3zq", "assetId": 0, "balance": 0.00003628}, + { "address": "QXmYaDzKQdGiAMncJCr1FqXy6tX3avMRm9", "assetId": 0, "balance": 0.00003632}, + { "address": "QXmYaDzKQdGiAMncJCr1FqXy6tX3avMRm9", "assetId": 2, "balance": 0.00000004}, + { "address": "QXNzWKLR9pqHW5KCUCFvcaUTwWKWvdYhzi", "assetId": 0, "balance": -0.00010321}, + { "address": "QXqc1jH2DL6H3qFCHxTBABivtkqJsoBYyQ", "assetId": 0, "balance": 0.00003628}, + { "address": "QXrkYRBJkp3CQ2ryjvWskszuWTXRRLbhTB", "assetId": 0, "balance": -0.00010321}, + { "address": "QXUWuZ2oAUodMU8EAQkAkDwkQHS1SFxpts", "assetId": 0, "balance": 0.00000011}, + { "address": "QXUWuZ2oAUodMU8EAQkAkDwkQHS1SFxpts", "assetId": 2, "balance": 0.00000011}, + { "address": "QXV2EabmW5AqDa4usWyv13QvxtkSUF5LFs", "assetId": 0, "balance": -0.00010321}, + { "address": "QXXfBJz9UfAgTEAn3b9W9jxmJYtqar1P78", "assetId": 0, "balance": 0.00003628}, + { "address": "QXYk68x2tiUrDBv8eq6wd4KtBmLHYiC4zR", "assetId": 0, "balance": 0.00003628}, + { "address": "QXYY8BwyYn71nDg9UKKMveyTLPZWErrtaT", "assetId": 0, "balance": -0.00010321}, + { "address": "QXZ3Uqs3KcfZDFgCURso8upzmPxxHtD9rT", "assetId": 0, "balance": 0.00003628}, + { "address": "QXzpeNdKwoxycyqZ2UanqFGngCN72nYygj", "assetId": 0, "balance": 0.00003628}, + { "address": "QY1RFZTD2ogRohf3UrdT4g1Qo9D122AZDN", "assetId": 0, "balance": 0.00000011}, + { "address": "QY1rJEuFJ5C6vp6QPczQbwUpg2F8KdPRoE", "assetId": 0, "balance": -0.00010321}, + { "address": "QY4xWXWbyU4t2zrcpZUTAR3kcXpcXw63Qn", "assetId": 0, "balance": 0.00003628}, + { "address": "QY6ZGZdi8h5op2VrRXkG1W5Jp3feLwp7ZD", "assetId": 0, "balance": 0.00003628}, + { "address": "QY82MasqEH6ChwXaETH4piMtE8Pk4NBWD3", "assetId": 0, "balance": 0.00000103}, + { "address": "QYA1jbLKSSY1q1Zo2VwuDe6vTJXQrr1wu1", "assetId": 0, "balance": -0.00010321}, + { "address": "QYAbYY1mVfCcXSoWPmVzUhvh7UBaK5enER", "assetId": 0, "balance": 0.00003628}, + { "address": "QYAYB8m9CfksGvEjGnj49q74bNDCGGaZqV", "assetId": 0, "balance": 0.00003628}, + { "address": "QYB9dJizBvcsmhFBgB4tzLAKsQvb5HQggo", "assetId": 0, "balance": 0.00003628}, + { "address": "QYcEBpuJ9RmdFGX6cdKAjSwnNVhwbtFLdr", "assetId": 0, "balance": 0.00003628}, + { "address": "QYD3kXchZ86vUyJBXNCVQ4LUvTAd6PUZW3", "assetId": 0, "balance": 0.00003628}, + { "address": "QYGcPZcRhGaY1MsiDr3VtwTXmB9TAbLFSn", "assetId": 0, "balance": 0.00000011}, + { "address": "QYGNMWBmqWgVtMWGHypAsKhDVQw5mrFZww", "assetId": 0, "balance": 0.00000011}, + { "address": "QYGrsQT4yhRUxiKfVgo8M5Sovfy1zcjUsr", "assetId": 0, "balance": 0.00003628}, + { "address": "QYgVi26jUqMzJo4ahZV9yekQNnYKHBaX8r", "assetId": 0, "balance": 0.00000132}, + { "address": "QYgVi26jUqMzJo4ahZV9yekQNnYKHBaX8r", "assetId": 2, "balance": 0.00000092}, + { "address": "QYHvrW3bwYFeMTUEYascXhXkBAzUkcGbqn", "assetId": 0, "balance": 0.00003628}, + { "address": "QYicTvqqPFt7buJfRd9cgs2xvJ2rnvyTzX", "assetId": 0, "balance": 0.00003628}, + { "address": "QYkpsUvut4eufXqJUnUbCzDajK5RyQ1Vzg", "assetId": 0, "balance": 0.00003628}, + { "address": "QYn2Uh4eii4SE29BpEPeRySbAeb9R6tGbf", "assetId": 0, "balance": 0.00003628}, + { "address": "QYoNKJhgva9ECexhgAmB3r4ucM8xwJbTWu", "assetId": 0, "balance": 0.00003628}, + { "address": "QYp5W4kGvCHfzeCgDyoCAWBZ9gViECNS5J", "assetId": 0, "balance": 0.00003628}, + { "address": "QYphDYA1te9acFNc7FEmFBu3FTTomp4ATZ", "assetId": 0, "balance": 0.00003628}, + { "address": "QYppKiFc7zt1EDXy1dUwHMHsnm2ckVsHTc", "assetId": 0, "balance": 0.00002352}, + { "address": "QYREQw3ohthywupqzLBRMjGkRSvbFPLBow", "assetId": 0, "balance": -0.00010321}, + { "address": "QYsh2NB6TogqV1iXHmHXcVaWw25WEYA94o", "assetId": 0, "balance": 0.00000011}, + { "address": "QYsh2NB6TogqV1iXHmHXcVaWw25WEYA94o", "assetId": 2, "balance": 0.00000011}, + { "address": "QYt1n7g68RrttdF8BdnqZkkKcYq1RHTeBF", "assetId": 0, "balance": 0.00003628}, + { "address": "QYTDS3XqzHWcqmhXTsDcUDVAbQVXXVaVVs", "assetId": 0, "balance": -0.00010321}, + { "address": "QYTmTSxB8GdnruZWA7Dvod9ihRQrAiLxn1", "assetId": 0, "balance": 0.00001276}, + { "address": "QYZj8LFUwQVKZGBLe9FTgWnf6pLEyJJDZi", "assetId": 0, "balance": 0.00003628}, + { "address": "QZ19JRpSsgvm4z6EjnbhdxJBoUYzDGvP3x", "assetId": 0, "balance": 0.00003628}, + { "address": "QZ1iWoraqiezeAHrgTsC13MTcrwHJdRwgk", "assetId": 0, "balance": -0.00010321}, + { "address": "QZ2gi6BhUNpGmrErgJLFuY1WHy6xK1J7qX", "assetId": 0, "balance": 0.00003628}, + { "address": "QZ7wvWAUcHKRhvQ3ijdrqM4zucQKCgQ1hQ", "assetId": 0, "balance": 0.00003628}, + { "address": "QZb1jPdakvcB9f7aVRU3wLXRcixgk8tPdU", "assetId": 0, "balance": 0.00003628}, + { "address": "QZb81oH9N6M4ZjPstDJuceARrdjLi8dY1x", "assetId": 0, "balance": 0.00003628}, + { "address": "QZbBJuoU892QYTQ4N1sJT9bVE3HNNdSw55", "assetId": 0, "balance": -0.00003608}, + { "address": "QZBTByprtp1MGQbEND6H95cPsrGaKEJEmy", "assetId": 0, "balance": 0.00001276}, + { "address": "QZCauGWyChXBgEQiXAJLmSaaz94Asgi8wU", "assetId": 0, "balance": 0.00003628}, + { "address": "QZgpMDQeZ3ReC13wnTvP94hVJoyAgVXEs6", "assetId": 0, "balance": 0.00003628}, + { "address": "QZHC8bBNbHTSEmdjKMQJFHAJhRwrTBXje1", "assetId": 0, "balance": 0.00003628}, + { "address": "QZiMCvxMJqG3bG6SsET43zegm4mtm2TABA", "assetId": 0, "balance": 0.00003628}, + { "address": "QZiYh4m4Uh3FH52cnow8MrNyXhSH88bp2H", "assetId": 0, "balance": 0.00003639}, + { "address": "QZiYh4m4Uh3FH52cnow8MrNyXhSH88bp2H", "assetId": 2, "balance": 0.00000011}, + { "address": "QZJc1V32oFm8tufB4bk7fa3aepu4EdkeDU", "assetId": 0, "balance": 0.00003628}, + { "address": "QZjCgcSVvSRsFZeLJz9C5dTa36s3cSKqvB", "assetId": 0, "balance": 0.00001276}, + { "address": "QZKKSYCnTaB56dT1dkXiV86eU6Pc9ADos2", "assetId": 0, "balance": 0.00003628}, + { "address": "QZkThgFfExognAbxLjYZGVHpL7X6g3EG4A", "assetId": 0, "balance": 0.00000026}, + { "address": "QZkThgFfExognAbxLjYZGVHpL7X6g3EG4A", "assetId": 2, "balance": 0.00000015}, + { "address": "QZMhBGpATjZ9ZK3fdcRvXW3RWKAAETymQa", "assetId": 0, "balance": 0.00001276}, + { "address": "QZMzF4iTBV93LP5Vkv7Ka3Q2xjdUwcUhcV", "assetId": 0, "balance": 0.00000103}, + { "address": "QZNDNgaBJhkUjtb66hGvjAs3s1V2TESDxE", "assetId": 0, "balance": 0.00003628}, + { "address": "QZpCNquYLc5B6xiUwsPtMB7M6f1CWcLBwP", "assetId": 0, "balance": 0.00003628}, + { "address": "QZqfJg1raAA3AzuivGD6sCQfQQcekAM6tx", "assetId": 0, "balance": 0.00003628}, + { "address": "QZRBPwvFBv59rZ4MzuPnjVi7cq5Uv7WqqR", "assetId": 0, "balance": 0.00003628}, + { "address": "QZtebdFopCUyGBQs5S5WYPckPcVZh19E4q", "assetId": 0, "balance": -0.00010321}, + { "address": "QZvHW7amu5DNktsBgaMrR1brHZhhhVwKLW", "assetId": 0, "balance": 0.00000011}, + { "address": "QZw7tgMttSySNMKfcMrEbdtnqHVrQ9w9fT", "assetId": 0, "balance": 0.00003639}, + { "address": "QZw7tgMttSySNMKfcMrEbdtnqHVrQ9w9fT", "assetId": 2, "balance": 0.00000011}, + { "address": "QZWL5atv3jQi3SdcQPS91vGhbk4Mi5CF8z", "assetId": 0, "balance": 0.00001276}, + { "address": "QZxNth97o4UNw6XbDY7fnykzuKaxmmqaR1", "assetId": 0, "balance": 0.00003628}, + { "address": "QZXo75Sk5AHuuuRX4VcHCBvcHaCHGHBVa2", "assetId": 0, "balance": -0.00010321}, + { "address": "QZzxwbQZ7Gi4kSVa39bcXb3q12AhGRQDXA", "assetId": 0, "balance": -0.00010321} +] diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index b94b291b3..17858d8de 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -4,15 +4,19 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.001", + "nameRegistrationUnitFee": "5", + "nameRegistrationUnitFeeTimestamp": 1645372800000, "useBrokenMD160ForAddresses": false, "requireGroupForApproval": false, "defaultGroupId": 0, "oneNamePerAccount": true, + "minAccountLevelToMint": 1, + "minAccountLevelForBlockSubmissions": 5, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 6, "founderEffectiveMintingLevel": 10, - "onlineAccountSignaturesMinLifetime": 2592000000, - "onlineAccountSignaturesMaxLifetime": 3196800000, + "onlineAccountSignaturesMinLifetime": 43200000, + "onlineAccountSignaturesMaxLifetime": 86400000, "rewardsByHeight": [ { "height": 1, "reward": 5.00 }, { "height": 259201, "reward": 4.75 }, @@ -48,6 +52,11 @@ "minutesPerBlock": 1 }, "featureTriggers": { + "atFindNextTransactionFix": 275000, + "newBlockSigHeight": 320000, + "shareBinFix": 399000, + "calcChainWeightTimestamp": 1620579600000, + "transactionV5Timestamp": 1642176000000 }, "genesisInfo": { "version": 4, diff --git a/src/main/resources/i18n/ApiError_de.properties b/src/main/resources/i18n/ApiError_de.properties index b2825e0d1..9ec2cd6db 100644 --- a/src/main/resources/i18n/ApiError_de.properties +++ b/src/main/resources/i18n/ApiError_de.properties @@ -1,14 +1,84 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# Keys are from api.ApiError enum -INVALID_ADDRESS = ung\u00FCltige adresse +# "localeLang": "de", -INVALID_ASSET_ID = ung\u00FCltige asset ID +### Common ### +JSON = JSON Nachricht konnte nicht geparst werden -INVALID_DATA = ung\u00FCltige daten +INSUFFICIENT_BALANCE = insufficient balance -INVALID_PUBLIC_KEY = ung\u00FCltiger public key +UNAUTHORIZED = API-Aufruf nicht autorisiert -INVALID_SIGNATURE = ung\u00FCltige signatur +REPOSITORY_ISSUE = Repository-Fehler -JSON = JSON nachricht konnte nicht geparsed werden +NON_PRODUCTION = this API call is not permitted for production systems + +BLOCKCHAIN_NEEDS_SYNC = blockchain needs to synchronize first + +NO_TIME_SYNC = noch keine Uhrensynchronisation + +### Validation ### +INVALID_SIGNATURE = ungültige Signatur + +INVALID_ADDRESS = ungültige Adresse + +INVALID_PUBLIC_KEY = ungültiger public key + +INVALID_DATA = ungültige Daten + +INVALID_NETWORK_ADDRESS = ungültige Netzwerk Adresse + +ADDRESS_UNKNOWN = Account Adresse unbekannt + +INVALID_CRITERIA = ungültige Suchkriterien + +INVALID_REFERENCE = ungültige Referenz + +TRANSFORMATION_ERROR = konnte JSON nicht in eine Transaktion umwandeln + +INVALID_PRIVATE_KEY = ungültiger private key + +INVALID_HEIGHT = ungültige block height + +CANNOT_MINT = Account kann nicht minten + +### Blocks ### +BLOCK_UNKNOWN = block unbekannt + +### Transactions ### +TRANSACTION_UNKNOWN = Transaktion unbekannt PUBLIC_KEY_NOT_FOUND = public key wurde nicht gefunden + +# this one is special in that caller expected to pass two additional strings, hence the two %s +TRANSACTION_INVALID = Transaktion ungültig: %s (%s) + +### Naming ### +NAME_UNKNOWN = Name unbekannt + +### Asset ### +INVALID_ASSET_ID = ungültige asset ID + +INVALID_ORDER_ID = ungültige asset order ID + +ORDER_UNKNOWN = unbekannte asset order ID + +### Groups ### +GROUP_UNKNOWN = Gruppe unbekannt + +### Foreign Blockchain ### +FOREIGN_BLOCKCHAIN_NETWORK_ISSUE = foreign blokchain or ElectrumX network issue + +FOREIGN_BLOCKCHAIN_BALANCE_ISSUE = insufficient balance on foreign blockchain + +FOREIGN_BLOCKCHAIN_TOO_SOON = too soon to broadcast foreign blockchain transaction (LockTime/median block time) + +### Trade Portal ### +ORDER_SIZE_TOO_SMALL = order amount too low + +### Data ### +FILE_NOT_FOUND = Datei nicht gefunden + +NO_REPLY = peer did not reply with data + diff --git a/src/main/resources/i18n/ApiError_en.properties b/src/main/resources/i18n/ApiError_en.properties index 6b72c8fcd..dfe73eef6 100644 --- a/src/main/resources/i18n/ApiError_en.properties +++ b/src/main/resources/i18n/ApiError_en.properties @@ -1,57 +1,87 @@ #Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) # Keys are from api.ApiError enum -ADDRESS_UNKNOWN = account address unknown +# "localeLang": "en", -BLOCKCHAIN_NEEDS_SYNC = blockchain needs to synchronize first +### Common ### +JSON = failed to parse JSON message -# Blocks -BLOCK_UNKNOWN = block unknown +INSUFFICIENT_BALANCE = insufficient balance -CANNOT_MINT = account cannot mint +UNAUTHORIZED = API call unauthorized -GROUP_UNKNOWN = group unknown +REPOSITORY_ISSUE = repository error -INVALID_ADDRESS = invalid address +NON_PRODUCTION = this API call is not permitted for production systems -# Assets -INVALID_ASSET_ID = invalid asset id +BLOCKCHAIN_NEEDS_SYNC = blockchain needs to synchronize first -INVALID_CRITERIA = invalid search criteria +NO_TIME_SYNC = no clock synchronization yet -INVALID_DATA = invalid data +### Validation ### +INVALID_SIGNATURE = invalid signature -INVALID_HEIGHT = invalid block height +INVALID_ADDRESS = invalid address + +INVALID_PUBLIC_KEY = invalid public key + +INVALID_DATA = invalid data INVALID_NETWORK_ADDRESS = invalid network address -INVALID_ORDER_ID = invalid asset order ID +ADDRESS_UNKNOWN = account address unknown + +INVALID_CRITERIA = invalid search criteria + +INVALID_REFERENCE = invalid reference + +TRANSFORMATION_ERROR = could not transform JSON into transaction INVALID_PRIVATE_KEY = invalid private key -INVALID_PUBLIC_KEY = invalid public key +INVALID_HEIGHT = invalid block height -INVALID_REFERENCE = invalid reference +CANNOT_MINT = account cannot mint -# Validation -INVALID_SIGNATURE = invalid signature +### Blocks ### +BLOCK_UNKNOWN = block unknown -JSON = failed to parse json message +### Transactions ### +TRANSACTION_UNKNOWN = transaction unknown + +PUBLIC_KEY_NOT_FOUND = public key not found +# this one is special in that caller expected to pass two additional strings, hence the two %s +TRANSACTION_INVALID = transaction invalid: %s (%s) + +### Naming ### NAME_UNKNOWN = name unknown -NON_PRODUCTION = this API call is not permitted for production systems +### Asset ### +INVALID_ASSET_ID = invalid asset ID + +INVALID_ORDER_ID = invalid asset order ID ORDER_UNKNOWN = unknown asset order ID -PUBLIC_KEY_NOT_FOUND = public key not found +### Groups ### +GROUP_UNKNOWN = group unknown -REPOSITORY_ISSUE = repository error +### Foreign Blockchain ### +FOREIGN_BLOCKCHAIN_NETWORK_ISSUE = foreign blokchain or ElectrumX network issue -TRANSACTION_INVALID = transaction invalid: %s (%s) +FOREIGN_BLOCKCHAIN_BALANCE_ISSUE = insufficient balance on foreign blockchain -TRANSACTION_UNKNOWN = transaction unknown +FOREIGN_BLOCKCHAIN_TOO_SOON = too soon to broadcast foreign blockchain transaction (LockTime/median block time) -TRANSFORMATION_ERROR = could not transform JSON into transaction +### Trade Portal ### +ORDER_SIZE_TOO_SMALL = order amount too low -UNAUTHORIZED = API call unauthorized +### Data ### +FILE_NOT_FOUND = file not found + +ORDER_SIZE_TOO_SMALL = order size too small + +FILE_NOT_FOUND = file not found + +NO_REPLY = peer didn't reply within the allowed time diff --git a/src/main/resources/i18n/ApiError_fi.properties b/src/main/resources/i18n/ApiError_fi.properties new file mode 100644 index 000000000..f95187003 --- /dev/null +++ b/src/main/resources/i18n/ApiError_fi.properties @@ -0,0 +1,86 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# Keys are from api.ApiError enum + +# Kielen muuttaminen suomeksi tapahtuu settings.json-tiedostossa +# +# "localeLang": "fi", +# muista pilkku lopussa jos komento ei ole viimeisellä rivillä + +### Common ### +JSON = JSON-viestin jaottelu epäonnistui + +INSUFFICIENT_BALANCE = insufficient balance + +UNAUTHORIZED = luvaton API-kutsu + +REPOSITORY_ISSUE = tietovarantovirhe (repo) + +NON_PRODUCTION = tämä API-kutsu on kielletty tuotantoversiossa + +BLOCKCHAIN_NEEDS_SYNC = lohkoketjun tarvitsee ensin synkronisoitua + +NO_TIME_SYNC = kello vielä synkronisoimatta + +### Validation ### +INVALID_SIGNATURE = kelvoton allekirjoitus + +INVALID_ADDRESS = osoite on kelvoton + +INVALID_PUBLIC_KEY = kelvoton julkinen avain + +INVALID_DATA = kelvoton data + +INVALID_NETWORK_ADDRESS = kelvoton verkko-osoite + +ADDRESS_UNKNOWN = tilin osoite on tuntematon + +INVALID_CRITERIA = kelvoton hakuehto + +INVALID_REFERENCE = kelvoton viite + +TRANSFORMATION_ERROR = JSON:in muuntaminen transaktioksi epäonnistui + +INVALID_PRIVATE_KEY = kelvoton yksityinen avain + +INVALID_HEIGHT = kelvoton lohkon korkeus + +CANNOT_MINT = tili ei voi lyödä rahaa + +### Blocks ### +BLOCK_UNKNOWN = tuntematon lohko + +### Transactions ### +TRANSACTION_UNKNOWN = tuntematon transaktio + +PUBLIC_KEY_NOT_FOUND = julkista avainta ei löytynyt + +# this one is special in that caller expected to pass two additional strings, hence the two %s +TRANSACTION_INVALID = kelvoton transaktio: %s (%s) + +### Naming ### +NAME_UNKNOWN = tuntematon nimi + +### Asset ### +INVALID_ASSET_ID = kelvoton ID resurssille + +INVALID_ORDER_ID = kelvoton resurssin tilaus-ID + +ORDER_UNKNOWN = tuntematon resurssin tilaus-ID + +### Groups ### +GROUP_UNKNOWN = tuntematon ryhmä + +### Foreign Blockchain ### +FOREIGN_BLOCKCHAIN_NETWORK_ISSUE = foreign blokchain or ElectrumX network issue + +FOREIGN_BLOCKCHAIN_BALANCE_ISSUE = insufficient balance on foreign blockchain + +FOREIGN_BLOCKCHAIN_TOO_SOON = too soon to broadcast foreign blockchain transaction (LockTime/median block time) + +### Trade Portal ### +ORDER_SIZE_TOO_SMALL = order amount too low + +### Data ### +FILE_NOT_FOUND = file not found + +NO_REPLY = peer did not reply with data \ No newline at end of file diff --git a/src/main/resources/i18n/ApiError_fr.properties b/src/main/resources/i18n/ApiError_fr.properties new file mode 100644 index 000000000..122b6d038 --- /dev/null +++ b/src/main/resources/i18n/ApiError_fr.properties @@ -0,0 +1,55 @@ +### Commun ### +JSON = échec de l'analyse du message JSON +INSUFFICIENT_BALANCE = balance insuffisante +UNAUTHORIZED = appel de l’API non autorisé +REPOSITORY_ISSUE = erreur de dépôt +NON_PRODUCTION = cet appel API n'est pas autorisé pour les systèmes en production +BLOCKCHAIN_NEEDS_SYNC = la blockchain doit d'abord être synchronisée +NO_TIME_SYNC = heure pas encore synchronisée + +### Validation ### +INVALID_SIGNATURE = signature invalide +INVALID_ADDRESS = adresse invalide +INVALID_PUBLIC_KEY = clé publique invalide +INVALID_DATA = données invalides +INVALID_NETWORK_ADDRESS = adresse réseau invalide +ADDRESS_UNKNOWN = adresse de compte inconnue +INVALID_CRITERIA = critère de recherche invalide +INVALID_REFERENCE = référence invalide +TRANSFORMATION_ERROR = ne peut pas transformer JSON en transaction +INVALID_PRIVATE_KEY = clé privée invalide +INVALID_HEIGHT = hauteur de bloc invalide +CANNOT_MINT = le compte ne peut pas mint + +### Blocks ### +BLOCK_UNKNOWN = bloc inconnu + +### Transactions ### +TRANSACTION_UNKNOWN = opération inconnue +PUBLIC_KEY_NOT_FOUND = clé publique introuvable + +# celui-ci est spécial dans le sens où l'appelant doit passer deux chaînes supplémentaires, d'où les deux %s +TRANSACTION_INVALID = transaction invalide: %s (%s) + +### Nommage ### +NAME_UNKNOWN = nom inconnu + +### Asset ### +INVALID_ASSET_ID = identifiant d'actif invalide +INVALID_ORDER_ID = identifiant de commande d'actif non valide +ORDER_UNKNOWN = identifiant d'ordre d'actif inconnu + +### Groupes ### +GROUP_UNKNOWN = groupe inconnu + +### Blockchain étrangère ### +FOREIGN_BLOCKCHAIN_NETWORK_ISSUE = Problème blokchain étrangère ou de réseau ElectrumX +FOREIGN_BLOCKCHAIN_BALANCE_ISSUE = solde insuffisant sur la blockchain étrangère +FOREIGN_BLOCKCHAIN_TOO_SOON = trop tôt pour diffuser la transaction sur la blockchain étrangère (temps de verrouillage/temps de bloc médian) + +### Portail de trading ### +ORDER_SIZE_TOO_SMALL = montant de commande trop bas + +### Données ### +FILE_NOT_FOUND = fichier introuvable +NO_REPLY = le pair n'a pas renvoyé de données \ No newline at end of file diff --git a/src/main/resources/i18n/ApiError_hu.properties b/src/main/resources/i18n/ApiError_hu.properties new file mode 100644 index 000000000..8aa783da6 --- /dev/null +++ b/src/main/resources/i18n/ApiError_hu.properties @@ -0,0 +1,86 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# Keys are from api.ApiError enum + +# Magyar myelvre forditotta: Szkíta (Scythian). 2021 Augusztus 7. +# Az alkalmazás nyelvének magyarra való változtatása a settings.json oldalon történik. + +# "localeLang": "hu", + +### Common ### +JSON = nem sikerült elemezni a JSON üzenetet + +INSUFFICIENT_BALANCE = elégtelen egyenleg + +UNAUTHORIZED = nem engedélyezett API-hívás + +REPOSITORY_ISSUE = adattári hiba + +NON_PRODUCTION = ez az API-hívás nem engedélyezett korlátozott rendszereken + +BLOCKCHAIN_NEEDS_SYNC = a blokkláncnak még szinkronizálnia kell + +NO_TIME_SYNC = az óraszinkronizálás még nem történt meg + +### Validation ### +INVALID_SIGNATURE = érvénytelen aláírás + +INVALID_ADDRESS = érvénytelen fiók cím + +INVALID_PUBLIC_KEY = érvénytelen nyilvános kulcs + +INVALID_DATA = érvénytelen adat + +INVALID_NETWORK_ADDRESS = érvénytelen hálózat cím + +ADDRESS_UNKNOWN = ismeretlen fiók cím + +INVALID_CRITERIA = érvénytelen keresési feltétel + +INVALID_REFERENCE = érvénytelen hivatkozás + +TRANSFORMATION_ERROR = nem sikerült tranzakcióvá alakítani a JSON-t + +INVALID_PRIVATE_KEY = érvénytelen privát kulcs + +INVALID_HEIGHT = érvénytelen blokkmagasság + +CANNOT_MINT = ez a fiók még nem tud QORT-ot verni + +### Blocks ### +BLOCK_UNKNOWN = ismeretlen blokk + +### Transactions ### +TRANSACTION_UNKNOWN = ismeretlen tranzakció + +PUBLIC_KEY_NOT_FOUND = nyilvános kulcs nem található + +# this one is special in that caller expected to pass two additional strings, hence the two %s +TRANSACTION_INVALID = érvénytelen tranzakció: %s (%s) + +### Naming ### +NAME_UNKNOWN = ismeretlen név + +### Asset ### +INVALID_ASSET_ID = érvénytelen eszközazonosító + +INVALID_ORDER_ID = érvénytelen eszközrendelési azonosító + +ORDER_UNKNOWN = ismeretlen eszközrendelési azonosító + +### Groups ### +GROUP_UNKNOWN = ismeretlen csoport + +### Foreign Blockchain ### +FOREIGN_BLOCKCHAIN_NETWORK_ISSUE = idegen blokklánc vagy ElectrumX hálózati probléma + +FOREIGN_BLOCKCHAIN_BALANCE_ISSUE = elégtelen egyenleg az idegen blokkláncon + +FOREIGN_BLOCKCHAIN_TOO_SOON = túl korai meghírdetni az idegen blokkláncon való tranzakciót (LockTime/medián blokkidő) + +### Trade Portal ### +ORDER_SIZE_TOO_SMALL = rendelési összeg túl alacsony + +### Data ### +FILE_NOT_FOUND = fájl nem található + +NO_REPLY = a másik csomópont nem válaszolt \ No newline at end of file diff --git a/src/main/resources/i18n/ApiError_it.properties b/src/main/resources/i18n/ApiError_it.properties new file mode 100644 index 000000000..339932002 --- /dev/null +++ b/src/main/resources/i18n/ApiError_it.properties @@ -0,0 +1,87 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# Keys are from api.ApiError enum +# Italian translation by Pabs 2021 + +# La modifica della lingua dell'UI è fatta nel file Settings.json +# +# "localeLang": "it", +# Si prega ricordare la virgola alla fine, se questo comando non è sull'ultima riga + +### Common ### +JSON = Impossibile analizzare il messaggio JSON + +INSUFFICIENT_BALANCE = insufficient balance + +UNAUTHORIZED = Chiamata API non autorizzata + +REPOSITORY_ISSUE = errore del repositorio + +NON_PRODUCTION = questa chiamata API non è consentita per i sistemi di produzione + +BLOCKCHAIN_NEEDS_SYNC = blockchain deve prima sincronizzarsi + +NO_TIME_SYNC = nessuna sincronizzazione dell'orologio ancora + +### Validation ### +INVALID_SIGNATURE = firma non valida + +INVALID_ADDRESS = indirizzo non valido + +INVALID_PUBLIC_KEY = chiave pubblica non valida + +INVALID_DATA = dati non validi + +INVALID_NETWORK_ADDRESS = indirizzo di rete non valido + +ADDRESS_UNKNOWN = indirizzo account sconosciuto + +INVALID_CRITERIA = criteri di ricerca non validi + +INVALID_REFERENCE = riferimento non valido + +TRANSFORMATION_ERROR = non è stato possibile trasformare JSON in transazione + +INVALID_PRIVATE_KEY = chiave privata non valida + +INVALID_HEIGHT = altezza blocco non valida + +CANNOT_MINT = l'account non può coniare + +### Blocks ### +BLOCK_UNKNOWN = blocco sconosciuto + +### Transactions ### +TRANSACTION_UNKNOWN = transazione sconosciuta + +PUBLIC_KEY_NOT_FOUND = chiave pubblica non trovata + +# this one is special in that caller expected to pass two additional strings, hence the two %s +TRANSACTION_INVALID = transazione non valida: %s (%s) + +### Naming ### +NAME_UNKNOWN = nome sconosciuto + +### Asset ### +INVALID_ASSET_ID = identificazione risorsa non valida + +INVALID_ORDER_ID = identificazione di ordine di risorsa non valida + +ORDER_UNKNOWN = identificazione di ordine di risorsa sconosciuta + +### Groups ### +GROUP_UNKNOWN = gruppo sconosciuto + +### Foreign Blockchain ### +FOREIGN_BLOCKCHAIN_NETWORK_ISSUE = foreign blokchain or ElectrumX network issue + +FOREIGN_BLOCKCHAIN_BALANCE_ISSUE = insufficient balance on foreign blockchain + +FOREIGN_BLOCKCHAIN_TOO_SOON = too soon to broadcast foreign blockchain transaction (LockTime/median block time) + +### Trade Portal ### +ORDER_SIZE_TOO_SMALL = order amount too low + +### Data ### +FILE_NOT_FOUND = file not found + +NO_REPLY = peer did not reply with data \ No newline at end of file diff --git a/src/main/resources/i18n/ApiError_nl.properties b/src/main/resources/i18n/ApiError_nl.properties new file mode 100644 index 000000000..5c54cf643 --- /dev/null +++ b/src/main/resources/i18n/ApiError_nl.properties @@ -0,0 +1,83 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# Keys are from api.ApiError enum + +# "localeLang": "nl", + +### Common ### +JSON = lezen van JSON bericht gefaald + +INSUFFICIENT_BALANCE = insufficient balance + +UNAUTHORIZED = ongeautoriseerde API call + +REPOSITORY_ISSUE = repository fout + +NON_PRODUCTION = deze API call is niet toegestaan voor productiesystemen + +BLOCKCHAIN_NEEDS_SYNC = blockchain dient eerst gesynchronizeerd te worden + +NO_TIME_SYNC = klok nog niet gesynchronizeerd + +### Validation ### +INVALID_SIGNATURE = ongeldige handtekening + +INVALID_ADDRESS = ongeldig adres + +INVALID_PUBLIC_KEY = ongeldige public key + +INVALID_DATA = ongeldige gegevens + +INVALID_NETWORK_ADDRESS = ongeldig netwerkadres + +ADDRESS_UNKNOWN = account adres onbekend + +INVALID_CRITERIA = ongeldige zoekcriteria + +INVALID_REFERENCE = ongeldige verwijzing + +TRANSFORMATION_ERROR = JSON kon niet omgezet worden in transactie + +INVALID_PRIVATE_KEY = ongeldige private key + +INVALID_HEIGHT = ongeldige blokhoogte + +CANNOT_MINT = account kan niet munten + +### Blocks ### +BLOCK_UNKNOWN = blok onbekend + +### Transactions ### +TRANSACTION_UNKNOWN = onbekende transactie + +PUBLIC_KEY_NOT_FOUND = public key niet gevonden + +# this one is special in that caller expected to pass two additional strings, hence the two %s +TRANSACTION_INVALID = ongeldige transactie: %s (%s) + +### Naming ### +NAME_UNKNOWN = onbekende naam + +### Asset ### +INVALID_ASSET_ID = ongeldige asset ID + +INVALID_ORDER_ID = ongeldige asset order ID + +ORDER_UNKNOWN = onbekende asset order ID + +### Groups ### +GROUP_UNKNOWN = onbekende groep + +### Foreign Blockchain ### +FOREIGN_BLOCKCHAIN_NETWORK_ISSUE = foreign blokchain or ElectrumX network issue + +FOREIGN_BLOCKCHAIN_BALANCE_ISSUE = insufficient balance on foreign blockchain + +FOREIGN_BLOCKCHAIN_TOO_SOON = too soon to broadcast foreign blockchain transaction (LockTime/median block time) + +### Trade Portal ### +ORDER_SIZE_TOO_SMALL = order amount too low + +### Data ### +FILE_NOT_FOUND = file not found + +NO_REPLY = peer did not reply with data \ No newline at end of file diff --git a/src/main/resources/i18n/ApiError_ru.properties b/src/main/resources/i18n/ApiError_ru.properties new file mode 100644 index 000000000..61948a2a3 --- /dev/null +++ b/src/main/resources/i18n/ApiError_ru.properties @@ -0,0 +1,83 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# Keys are from api.ApiError enum + +# "localeLang": "ru", + +### Common ### +JSON = не удалось разобрать сообщение json + +INSUFFICIENT_BALANCE = insufficient balance + +UNAUTHORIZED = вызов API не авторизован + +REPOSITORY_ISSUE = ошибка репозитория + +NON_PRODUCTION = этот вызов API не разрешен для производственных систем + +BLOCKCHAIN_NEEDS_SYNC = блокчейн должен сначала синхронизироваться + +NO_TIME_SYNC = no clock synchronization yet + +### Validation ### +INVALID_SIGNATURE = недействительная подпись + +INVALID_ADDRESS = неизвестный адрес + +INVALID_PUBLIC_KEY = недействительный открытый ключ + +INVALID_DATA = неверные данные + +INVALID_NETWORK_ADDRESS = неверный сетевой адрес + +ADDRESS_UNKNOWN = неизвестная учетная запись + +INVALID_CRITERIA = неверные критерии поиска + +INVALID_REFERENCE = неверная ссылка + +TRANSFORMATION_ERROR = не удалось преобразовать JSON в транзакцию + +INVALID_PRIVATE_KEY = неверный приватный ключ + +INVALID_HEIGHT = недопустимая высота блока + +CANNOT_MINT = аккаунт не может чеканить + +### Blocks ### +BLOCK_UNKNOWN = неизвестный блок + +### Transactions ### +TRANSACTION_UNKNOWN = транзакция неизвестна + +PUBLIC_KEY_NOT_FOUND = открытый ключ не найден + +# this one is special in that caller expected to pass two additional strings, hence the two %s +TRANSACTION_INVALID = транзакция недействительна: %s (%s) + +### Naming ### +NAME_UNKNOWN = имя неизвестно + +### Asset ### +INVALID_ASSET_ID = неверный идентификатор актива + +INVALID_ORDER_ID = неверный идентификатор заказа актива + +ORDER_UNKNOWN = неизвестный идентификатор заказа актива + +### Groups ### +GROUP_UNKNOWN = неизвестная группа + +### Foreign Blockchain ### +FOREIGN_BLOCKCHAIN_NETWORK_ISSUE = foreign blokchain or ElectrumX network issue + +FOREIGN_BLOCKCHAIN_BALANCE_ISSUE = insufficient balance on foreign blockchain + +FOREIGN_BLOCKCHAIN_TOO_SOON = too soon to broadcast foreign blockchain transaction (LockTime/median block time) + +### Trade Portal ### +ORDER_SIZE_TOO_SMALL = order amount too low + +### Data ### +FILE_NOT_FOUND = file not found + +NO_REPLY = peer did not reply with data \ No newline at end of file diff --git a/src/main/resources/i18n/SysTray_de.properties b/src/main/resources/i18n/SysTray_de.properties new file mode 100644 index 000000000..0f2a93dce --- /dev/null +++ b/src/main/resources/i18n/SysTray_de.properties @@ -0,0 +1,40 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# SysTray pop-up menu + +AUTO_UPDATE = Automatisches Update + +APPLYING_UPDATE_AND_RESTARTING = Automatisches Update anwenden und neu starten … + +BLOCK_HEIGHT = height + +BUILD_VERSION = Build-Version + +CHECK_TIME_ACCURACY = Prüfe Zeitgenauigkeit + +CONNECTING = Verbindung wird hergestellt + +CONNECTION = Verbindung + +CONNECTIONS = Verbindungen + +CREATING_BACKUP_OF_DB_FILES = Erstellen Backup von Datenbank Dateien … + +DB_BACKUP = Datenbank Backup + +DB_CHECKPOINT = Datenbank Kontrollpunkt + +EXIT = Verlassen + +MINTING_DISABLED = NOT minting + +MINTING_ENABLED = \u2714 Minting + +OPEN_UI = Öffne UI + +PERFORMING_DB_CHECKPOINT = Speichern nicht übergebener Datenbank Änderungen … + +SYNCHRONIZE_CLOCK = Synchronisiere Uhr + +SYNCHRONIZING_BLOCKCHAIN = Synchronisierung + +SYNCHRONIZING_CLOCK = Synchronisierung Uhr diff --git a/src/main/resources/i18n/SysTray_en.properties b/src/main/resources/i18n/SysTray_en.properties index f41c1a32a..07541339e 100644 --- a/src/main/resources/i18n/SysTray_en.properties +++ b/src/main/resources/i18n/SysTray_en.properties @@ -1,12 +1,14 @@ #Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) # SysTray pop-up menu -APPLYING_UPDATE_AND_RESTARTING = Applying automatic update and restarting... - AUTO_UPDATE = Auto Update +APPLYING_UPDATE_AND_RESTARTING = Applying automatic update and restarting... + BLOCK_HEIGHT = height +BUILD_VERSION = Build version + CHECK_TIME_ACCURACY = Check time accuracy CONNECTING = Connecting @@ -19,20 +21,21 @@ CREATING_BACKUP_OF_DB_FILES = Creating backup of database files... DB_BACKUP = Database Backup +DB_MAINTENANCE = Database Maintenance + +DB_CHECKPOINT = Database Checkpoint + EXIT = Exit MINTING_DISABLED = NOT minting MINTING_ENABLED = \u2714 Minting -# Nagging about lack of NTP time sync -NTP_NAG_CAPTION = Computer's clock is inaccurate! - -NTP_NAG_TEXT_UNIX = Install NTP service to get an accurate clock. +OPEN_UI = Open UI -NTP_NAG_TEXT_WINDOWS = Select "Synchronize clock" from menu to fix. +PERFORMING_DB_CHECKPOINT = Saving uncommitted database changes... -OPEN_UI = Open UI +PERFORMING_DB_MAINTENANCE = Performing scheduled maintenance... SYNCHRONIZE_CLOCK = Synchronize clock diff --git a/src/main/resources/i18n/SysTray_fi.properties b/src/main/resources/i18n/SysTray_fi.properties new file mode 100644 index 000000000..edd062bc6 --- /dev/null +++ b/src/main/resources/i18n/SysTray_fi.properties @@ -0,0 +1,44 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# SysTray pop-up menu + +AUTO_UPDATE = Automaattinen päivitys + +APPLYING_UPDATE_AND_RESTARTING = Automaattinen päivitys käynnissä, uudelleenkäynnistys seuraa... + +BLOCK_HEIGHT = korkeus + +BUILD_VERSION = Versio + +CHECK_TIME_ACCURACY = Tarkista ajan tarkkuus + +CONNECTING = Yhdistää + +CONNECTION = yhteys + +CONNECTIONS = yhteyttä + +CREATING_BACKUP_OF_DB_FILES = Luodaan varmuuskopio tietokannan tiedostoista... + +DB_BACKUP = Tietokannan varmuuskopio + +DB_MAINTENANCE = Database Maintenance + +DB_CHECKPOINT = Tietokannan varmistuspiste + +EXIT = Pois + +MINTING_DISABLED = EI lyö rahaa + +MINTING_ENABLED = \u2714 Lyö rahaa + +OPEN_UI = Avaa UI + +PERFORMING_DB_CHECKPOINT = Tallentaa kommittoidut tietokantamuutokset... + +PERFORMING_DB_MAINTENANCE = Performing scheduled maintenance... + +SYNCHRONIZE_CLOCK = Synkronisoi kello + +SYNCHRONIZING_BLOCKCHAIN = Synkronisoi + +SYNCHRONIZING_CLOCK = Synkronisoi kelloa diff --git a/src/main/resources/i18n/SysTray_fr.properties b/src/main/resources/i18n/SysTray_fr.properties new file mode 100644 index 000000000..b8aac433d --- /dev/null +++ b/src/main/resources/i18n/SysTray_fr.properties @@ -0,0 +1,41 @@ +AUTO_UPDATE = Mise à jour automatique + +APPLYING_UPDATE_AND_RESTARTING = Application de la mise à jour automatique et redémarrage... + +BLOCK_HEIGHT = hauteur + +BUILD_VERSION = Numéro de version + +CHECK_TIME_ACCURACY = Vérifier l'heure + +CONNECTING = Connexion en cours + +CONNECTION = connexion + +CONNECTIONS = connexions + +CREATING_BACKUP_OF_DB_FILES = Création d'une sauvegarde des fichiers de la base de données... + +DB_BACKUP = Sauvegarde de la base de données + +DB_MAINTENANCE = Maintenance de la base de données + +DB_CHECKPOINT = Point de contrôle de la base de données + +EXIT = Quitter + +MINTING_DISABLED = NE mint PAS + +MINTING_ENABLED = \u2714 Minting + +OPEN_UI = Ouvrir l'interface + +PERFORMING_DB_CHECKPOINT = Enregistrement des modifications de base de données non validées... + +PERFORMING_DB_MAINTENANCE = Entrain d'effectuer la maintenance programmée... + +SYNCHRONIZE_CLOCK = Mettre l'heure à jour + +SYNCHRONIZING_BLOCKCHAIN = Synchronisation + +SYNCHRONIZING_CLOCK = Synchronisation de l'heure diff --git a/src/main/resources/i18n/SysTray_hu.properties b/src/main/resources/i18n/SysTray_hu.properties new file mode 100644 index 000000000..be4bef250 --- /dev/null +++ b/src/main/resources/i18n/SysTray_hu.properties @@ -0,0 +1,46 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# SysTray pop-up menu + +# Magyar myelvre forditotta: Szkíta (Scythian). 2021 Augusztus 7. + +AUTO_UPDATE = Automatikus Frissítés + +APPLYING_UPDATE_AND_RESTARTING = Automatikus frissítés és újraindítás alkalmazása... + +BLOCK_HEIGHT = blokkmagasság + +BUILD_VERSION = Verzió + +CHECK_TIME_ACCURACY = Idő pontosság ellenőrzése + +CONNECTING = Kapcsolódás + +CONNECTION = kapcsolat + +CONNECTIONS = kapcsolat + +CREATING_BACKUP_OF_DB_FILES = Adatbázis fájlok biztonsági mentésének létrehozása... + +DB_BACKUP = Adatbázis biztonsági mentése + +DB_MAINTENANCE = Database Maintenance + +DB_CHECKPOINT = Adatbázis-ellenőrzőpont + +EXIT = Kilépés + +MINTING_DISABLED = QORT-érmeverés jelenleg nincs folyamatban + +MINTING_ENABLED = \u2714 QORT-érmeverés folyamatban + +OPEN_UI = Felhasználói eszköz megnyitása + +PERFORMING_DB_CHECKPOINT = Mentetlen adatbázis-módosítások mentése... + +PERFORMING_DB_MAINTENANCE = Performing scheduled maintenance... + +SYNCHRONIZE_CLOCK = Óra-szinkronizálás megkezdése + +SYNCHRONIZING_BLOCKCHAIN = Szinkronizálás + +SYNCHRONIZING_CLOCK = Óra-szinkronizálás folyamatban diff --git a/src/main/resources/i18n/SysTray_it.properties b/src/main/resources/i18n/SysTray_it.properties new file mode 100644 index 000000000..326c71c22 --- /dev/null +++ b/src/main/resources/i18n/SysTray_it.properties @@ -0,0 +1,45 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# SysTray pop-up menu +# Italian translation by Pabs 2021 + +APPLYING_UPDATE_AND_RESTARTING = Applicando aggiornamento automatico e riavviando... + +AUTO_UPDATE = Aggiornamento automatico + +BLOCK_HEIGHT = altezza + +BUILD_VERSION = Versione + +CHECK_TIME_ACCURACY = Controlla la precisione dell'ora + +CONNECTING = Collegando + +CONNECTION = connessione + +CONNECTIONS = connessioni + +CREATING_BACKUP_OF_DB_FILES = Creazione di backup dei file di database... + +DB_BACKUP = Backup del database + +DB_MAINTENANCE = Database Maintenance + +DB_CHECKPOINT = Punto di controllo del database + +EXIT = Uscita + +MINTING_DISABLED = NON coniando + +MINTING_ENABLED = \u2714 Coniando + +OPEN_UI = Apri UI + +PERFORMING_DB_CHECKPOINT = Salvataggio delle modifiche al database non salvate... + +PERFORMING_DB_MAINTENANCE = Performing scheduled maintenance... + +SYNCHRONIZE_CLOCK = Sincronizza orologio + +SYNCHRONIZING_BLOCKCHAIN = Sincronizzando + +SYNCHRONIZING_CLOCK = Sincronizzando orologio diff --git a/src/main/resources/i18n/SysTray_nl.properties b/src/main/resources/i18n/SysTray_nl.properties new file mode 100644 index 000000000..ddf1527f5 --- /dev/null +++ b/src/main/resources/i18n/SysTray_nl.properties @@ -0,0 +1,42 @@ +Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# SysTray pop-up menu + +APPLYING_UPDATE_AND_RESTARTING = Automatische update en herstart worden uitgevoerd... + +AUTO_UPDATE = Automatische Update + +BLOCK_HEIGHT = hoogte + +BUILD_VERSION = Versie + +CHECK_TIME_ACCURACY = Controleer accuraatheid van de tijd + +CONNECTING = Verbinden + +CONNECTION = verbinding + +CONNECTIONS = verbindingen + +CREATING_BACKUP_OF_DB_FILES = Backup van databasebestanden wordt gemaakt... + +DB_BACKUP = Database Backup + +DB_CHECKPOINT = Database Controlepunt + +EXIT = Verlaten + +MINTING_DISABLED = NIET muntend + +MINTING_ENABLED = \u2714 Muntend + +OPEN_UI = Open UI + +PERFORMING_DB_CHECKPOINT = Nieuwe veranderingen aan database worden opgeslagen... + +PERFORMING_DB_MAINTENANCE = Performing scheduled maintenance... + +SYNCHRONIZE_CLOCK = Synchronizeer klok + +SYNCHRONIZING_BLOCKCHAIN = Aan het synchronizeren + +SYNCHRONIZING_CLOCK = Klok wordt gesynchronizeerd diff --git a/src/main/resources/i18n/SysTray_ru.properties b/src/main/resources/i18n/SysTray_ru.properties new file mode 100644 index 000000000..c124b5005 --- /dev/null +++ b/src/main/resources/i18n/SysTray_ru.properties @@ -0,0 +1,42 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# SysTray pop-up menu + +APPLYING_UPDATE_AND_RESTARTING = Применение автоматического обновления и перезапуска... + +AUTO_UPDATE = Автоматическое обновление + +BLOCK_HEIGHT = Высота блока + +BUILD_VERSION = Build version + +CHECK_TIME_ACCURACY = Проверка точного времени + +CONNECTING = Подключение + +CONNECTION = Соединение + +CONNECTIONS = Соединений + +CREATING_BACKUP_OF_DB_FILES = Создание резервной копии файлов базы данных... + +DB_BACKUP = Резервное копирование базы данных + +DB_MAINTENANCE = Database Maintenance + +EXIT = Выход + +MINTING_DISABLED = Чеканка отключена + +MINTING_ENABLED = Чеканка активна + +OPEN_UI = Открыть пользовательский интерфейс + +PERFORMING_DB_CHECKPOINT = Saving uncommitted database changes... + +PERFORMING_DB_MAINTENANCE = Performing scheduled maintenance... + +SYNCHRONIZE_CLOCK = Синхронизировать время + +SYNCHRONIZING_BLOCKCHAIN = Синхронизация цепи + +SYNCHRONIZING_CLOCK = Проверка времени diff --git a/src/main/resources/i18n/SysTray_zh.properties b/src/main/resources/i18n/SysTray_zh.properties deleted file mode 100644 index bb2e14264..000000000 --- a/src/main/resources/i18n/SysTray_zh.properties +++ /dev/null @@ -1,31 +0,0 @@ -#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) -# SysTray pop-up menu - -BLOCK_HEIGHT = \u5757\u9AD8\u5EA6 - -CHECK_TIME_ACCURACY = \u68C0\u67E5\u65F6\u95F4\u51C6\u786E\u6027 - -CONNECTION = \u4E2A\u8FDE\u63A5 - -CONNECTIONS = \u4E2A\u8FDE\u63A5 - -EXIT = \u9000\u51FA\u8F6F\u4EF6 - -MINTING_DISABLED = \u6CA1\u6709\u94F8\u5E01 - -MINTING_ENABLED = \u2714 \u94F8\u5E01 - -# Nagging about lack of NTP time sync -NTP_NAG_CAPTION = \u7535\u8111\u7684\u65F6\u949F\u4E0D\u51C6\u786E\uFF01 - -NTP_NAG_TEXT_UNIX = \u5B89\u88C5NTP\u670D\u52A1\u4EE5\u83B7\u5F97\u51C6\u786E\u7684\u65F6\u949F\u3002 - -NTP_NAG_TEXT_WINDOWS = \u4ECE\u83DC\u5355\u4E2D\u9009\u62E9\u201C\u540C\u6B65\u65F6\u949F\u201D\u8FDB\u884C\u4FEE\u590D\u3002 - -OPEN_UI = \u5F00\u542F\u754C\u9762 - -SYNCHRONIZE_CLOCK = \u540C\u6B65\u65F6\u949F - -SYNCHRONIZING_BLOCKCHAIN = \u540C\u6B65\u533A\u5757\u94FE - -SYNCHRONIZING_CLOCK = \u540C\u6B65\u7740\u65F6\u949F diff --git a/src/main/resources/i18n/SysTray_zh_CN.properties b/src/main/resources/i18n/SysTray_zh_CN.properties new file mode 100644 index 000000000..6d8318e2a --- /dev/null +++ b/src/main/resources/i18n/SysTray_zh_CN.properties @@ -0,0 +1,42 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# SysTray pop-up menu + +AUTO_UPDATE = Auto Update + +APPLYING_UPDATE_AND_RESTARTING = Applying automatic update and restarting... + +BLOCK_HEIGHT = 区块高度 + +BUILD_VERSION = Build version + +CHECK_TIME_ACCURACY = 检查时间准确性 + +CONNECTING = Connecting + +CONNECTION = 个链接 + +CONNECTIONS = 个链接 + +CREATING_BACKUP_OF_DB_FILES = Creating backup of database files... + +DB_BACKUP = Database Backup + +DB_CHECKPOINT = Database Checkpoint + +EXIT = 退出核心 + +MINTING_DISABLED = 没有铸币 + +MINTING_ENABLED = ✔ 铸币 + +OPEN_UI = 开启Qortal界面 + +PERFORMING_DB_CHECKPOINT = Saving uncommitted database changes... + +PERFORMING_DB_MAINTENANCE = Performing scheduled maintenance... + +SYNCHRONIZE_CLOCK = 同步时钟 + +SYNCHRONIZING_BLOCKCHAIN = 正在同步区块链 + +SYNCHRONIZING_CLOCK = 正在同步时钟 diff --git a/src/main/resources/i18n/SysTray_zh_TW.properties b/src/main/resources/i18n/SysTray_zh_TW.properties new file mode 100644 index 000000000..3af0c84cd --- /dev/null +++ b/src/main/resources/i18n/SysTray_zh_TW.properties @@ -0,0 +1,40 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# SysTray pop-up menu + +AUTO_UPDATE = Auto Update + +APPLYING_UPDATE_AND_RESTARTING = Applying automatic update and restarting... + +BLOCK_HEIGHT = 區塊高度 + +BUILD_VERSION = Build version + +CHECK_TIME_ACCURACY = 檢查時間準確性 + +CONNECTING = Connecting + +CONNECTION = 個鏈接 + +CONNECTIONS = 個鏈接 + +CREATING_BACKUP_OF_DB_FILES = Creating backup of database files... + +DB_BACKUP = Database Backup + +DB_CHECKPOINT = Database Checkpoint + +EXIT = 退出核心 + +MINTING_DISABLED = 沒有鑄幣 + +MINTING_ENABLED = ✔ 鑄幣 + +OPEN_UI = 開啓Qortal界面 + +PERFORMING_DB_CHECKPOINT = Saving uncommitted database changes... + +SYNCHRONIZE_CLOCK = 同步時鐘 + +SYNCHRONIZING_BLOCKCHAIN = 正在同步區塊鏈 + +SYNCHRONIZING_CLOCK = 正在同步時鐘 \ No newline at end of file diff --git a/src/main/resources/i18n/TransactionValidity_en.properties b/src/main/resources/i18n/TransactionValidity_en.properties index d3530d467..7c4d18a13 100644 --- a/src/main/resources/i18n/TransactionValidity_en.properties +++ b/src/main/resources/i18n/TransactionValidity_en.properties @@ -1,176 +1,193 @@ +OK = OK -ACCOUNT_ALREADY_EXISTS = account already exists +INVALID_ADDRESS = invalid address -ACCOUNT_CANNOT_REWARD_SHARE = account cannot reward-share +NEGATIVE_AMOUNT = invalid/negative amount -ALREADY_GROUP_ADMIN = already group admin +NEGATIVE_FEE = invalid/negative fee -ALREADY_GROUP_MEMBER = already group member +NO_BALANCE = insufficient balance -ALREADY_VOTED_FOR_THAT_OPTION = already voted for that option +INVALID_REFERENCE = invalid reference -ASSET_ALREADY_EXISTS = asset already exists +INVALID_NAME_LENGTH = invalid name length -ASSET_DOES_NOT_EXIST = ASSET_DOES_NOT_EXIST +INVALID_VALUE_LENGTH = invalid 'value' length -ASSET_DOES_NOT_MATCH_AT = ASSET_DOES_NOT_MATCH_AT +NAME_ALREADY_REGISTERED = name already registered -ASSET_NOT_SPENDABLE = ASSET_NOT_SPENDABLE +NAME_DOES_NOT_EXIST = name does not exist -AT_ALREADY_EXISTS = AT_ALREADY_EXISTS +INVALID_NAME_OWNER = invalid name owner -AT_IS_FINISHED = AT_IS_FINISHED +NAME_ALREADY_FOR_SALE = name already for sale -AT_UNKNOWN = AT_UNKNOWN +NAME_NOT_FOR_SALE = name is not for sale -BANNED_FROM_GROUP = BANNED_FROM_GROUP +BUYER_ALREADY_OWNER = buyer is already owner -BAN_EXISTS = BAN_EXISTS +INVALID_AMOUNT = invalid amount -BAN_UNKNOWN = BAN_UNKNOWN +INVALID_SELLER = invalid seller -BUYER_ALREADY_OWNER = BUYER_ALREADY_OWNER +NAME_NOT_NORMALIZED = name not in Unicode 'normalized' form -CLOCK_NOT_SYNCED = CLOCK_NOT_SYNCED +INVALID_DESCRIPTION_LENGTH = invalid description length -DUPLICATE_OPTION = DUPLICATE_OPTION +INVALID_OPTIONS_COUNT = invalid options count -GROUP_ALREADY_EXISTS = GROUP_ALREADY_EXISTS +INVALID_OPTION_LENGTH = invalid options length -GROUP_APPROVAL_DECIDED = GROUP_APPROVAL_DECIDED +DUPLICATE_OPTION = duplicate option -GROUP_APPROVAL_NOT_REQUIRED = GROUP_APPROVAL_NOT_REQUIRED +POLL_ALREADY_EXISTS = poll already exists -GROUP_DOES_NOT_EXIST = GROUP_DOES_NOT_EXIST +POLL_DOES_NOT_EXIST = poll does not exist -GROUP_ID_MISMATCH = GROUP_ID_MISMATCH +POLL_OPTION_DOES_NOT_EXIST = poll option does not exist -GROUP_OWNER_CANNOT_LEAVE = GROUP_OWNER_CANNOT_LEAVE +ALREADY_VOTED_FOR_THAT_OPTION = already voted for that option -HAVE_EQUALS_WANT = HAVE_EQUALS_WANT +INVALID_DATA_LENGTH = invalid data length -INSUFFICIENT_FEE = INSUFFICIENT_FEE +INVALID_QUANTITY = invalid quantity -INVALID_ADDRESS = INVALID_ADDRESS +ASSET_DOES_NOT_EXIST = asset does not exist -INVALID_AMOUNT = INVALID_AMOUNT +INVALID_RETURN = invalid return -INVALID_ASSET_OWNER = INVALID_ASSET_OWNER +HAVE_EQUALS_WANT = have-asset is the same as want-asset -INVALID_AT_TRANSACTION = INVALID_AT_TRANSACTION +ORDER_DOES_NOT_EXIST = asset trade order does not exist -INVALID_AT_TYPE_LENGTH = INVALID_AT_TYPE_LENGTH +INVALID_ORDER_CREATOR = invalid order creator -INVALID_CREATION_BYTES = INVALID_CREATION_BYTES +INVALID_PAYMENTS_COUNT = invalid payments count -INVALID_DATA_LENGTH = INVALID_DATA_LENGTH +NEGATIVE_PRICE = invalid/negative price -INVALID_DESCRIPTION_LENGTH = INVALID_DESCRIPTION_LENGTH +INVALID_CREATION_BYTES = invalid creation bytes -INVALID_GROUP_APPROVAL_THRESHOLD = INVALID_GROUP_APPROVAL_THRESHOLD +INVALID_TAGS_LENGTH = invalid 'tags' length -INVALID_GROUP_ID = INVALID_GROUP_ID +INVALID_AT_TYPE_LENGTH = invalid AT 'type' length -INVALID_GROUP_OWNER = INVALID_GROUP_OWNER +INVALID_AT_TRANSACTION = invalid AT transaction -INVALID_LIFETIME = INVALID_LIFETIME +INSUFFICIENT_FEE = insufficient fee -INVALID_NAME_LENGTH = INVALID_NAME_LENGTH +ASSET_DOES_NOT_MATCH_AT = asset does not match AT's asset -INVALID_NAME_OWNER = INVALID_NAME_OWNER +ASSET_ALREADY_EXISTS = asset already exists -INVALID_OPTIONS_COUNT = INVALID_OPTIONS_COUNT +MISSING_CREATOR = missing creator -INVALID_OPTION_LENGTH = INVALID_OPTION_LENGTH +TIMESTAMP_TOO_OLD = timestamp too old -INVALID_ORDER_CREATOR = INVALID_ORDER_CREATOR +TIMESTAMP_TOO_NEW = timestamp too new -INVALID_PAYMENTS_COUNT = INVALID_PAYMENTS_COUNT +TOO_MANY_UNCONFIRMED = account has too many unconfirmed transactions pending -INVALID_PUBLIC_KEY = INVALID_PUBLIC_KEY +GROUP_ALREADY_EXISTS = group already exists -INVALID_QUANTITY = INVALID_QUANTITY +GROUP_DOES_NOT_EXIST = group does not exist -INVALID_REFERENCE = INVALID_REFERENCE +INVALID_GROUP_OWNER = invalid group owner -INVALID_RETURN = INVALID_RETURN +ALREADY_GROUP_MEMBER = already group member -INVALID_REWARD_SHARE_PERCENT = INVALID_REWARD_SHARE_PERCENT +GROUP_OWNER_CANNOT_LEAVE = group owner cannot leave group -INVALID_SELLER = INVALID_SELLER +NOT_GROUP_MEMBER = account is not a group member -INVALID_TAGS_LENGTH = INVALID_TAGS_LENGTH +ALREADY_GROUP_ADMIN = already group admin -INVALID_TX_GROUP_ID = INVALID_TX_GROUP_ID +NOT_GROUP_ADMIN = account is not a group admin -INVALID_VALUE_LENGTH = INVALID_VALUE_LENGTH +INVALID_LIFETIME = invalid lifetime -INVITE_UNKNOWN = INVITE_UNKNOWN +INVITE_UNKNOWN = group invite unknown -JOIN_REQUEST_EXISTS = JOIN_REQUEST_EXISTS +BAN_EXISTS = ban already exists -MAXIMUM_REWARD_SHARES = MAXIMUM_REWARD_SHARES +BAN_UNKNOWN = ban unknown -MISSING_CREATOR = MISSING_CREATOR +BANNED_FROM_GROUP = banned from group -MULTIPLE_NAMES_FORBIDDEN = MULTIPLE_NAMES_FORBIDDEN +JOIN_REQUEST_EXISTS = group join request already exists -NAME_ALREADY_FOR_SALE = NAME_ALREADY_FOR_SALE +INVALID_GROUP_APPROVAL_THRESHOLD = invalid group-approval threshold -NAME_ALREADY_REGISTERED = NAME_ALREADY_REGISTERED +GROUP_ID_MISMATCH = group ID mismatch -NAME_DOES_NOT_EXIST = NAME_DOES_NOT_EXIST +INVALID_GROUP_ID = invalid group ID -NAME_NOT_FOR_SALE = NAME_NOT_FOR_SALE +TRANSACTION_UNKNOWN = transaction unknown -NAME_NOT_LOWER_CASE = NAME_NOT_LOWER_CASE +TRANSACTION_ALREADY_CONFIRMED = transaction has already confirmed -NEGATIVE_AMOUNT = NEGATIVE_AMOUNT +INVALID_TX_GROUP_ID = invalid transaction group ID -NEGATIVE_FEE = NEGATIVE_FEE +TX_GROUP_ID_MISMATCH = transaction's group ID does not match -NEGATIVE_PRICE = NEGATIVE_PRICE +MULTIPLE_NAMES_FORBIDDEN = multiple registered names per account is forbidden -NOT_GROUP_ADMIN = NOT_GROUP_ADMIN +INVALID_ASSET_OWNER = invalid asset owner -NOT_GROUP_MEMBER = NOT_GROUP_MEMBER +AT_IS_FINISHED = AT has finished -NOT_MINTING_ACCOUNT = NOT_MINTING_ACCOUNT +NO_FLAG_PERMISSION = account does not have that permission -NOT_YET_RELEASED = NOT_YET_RELEASED +NOT_MINTING_ACCOUNT = account cannot mint -NO_BALANCE = NO_BALANCE +REWARD_SHARE_UNKNOWN = reward-share unknown -NO_BLOCKCHAIN_LOCK = node's blockchain currently busy +INVALID_REWARD_SHARE_PERCENT = invalid reward-share percent -NO_FLAG_PERMISSION = NO_FLAG_PERMISSION +PUBLIC_KEY_UNKNOWN = public key unknown -OK = OK +INVALID_PUBLIC_KEY = invalid public key + +AT_UNKNOWN = AT unknown + +AT_ALREADY_EXISTS = AT already exists + +GROUP_APPROVAL_NOT_REQUIRED = group-approval not required + +GROUP_APPROVAL_DECIDED = group-approval already decided + +MAXIMUM_REWARD_SHARES = already at maximum number of reward-shares for this account -ORDER_ALREADY_CLOSED = ORDER_ALREADY_CLOSED +TRANSACTION_ALREADY_EXISTS = transaction already exists -ORDER_DOES_NOT_EXIST = ORDER_DOES_NOT_EXIST +NO_BLOCKCHAIN_LOCK = node's blockchain currently busy + +ORDER_ALREADY_CLOSED = asset trade order is already closed + +CLOCK_NOT_SYNCED = clock not synchronized -POLL_ALREADY_EXISTS = POLL_ALREADY_EXISTS +ASSET_NOT_SPENDABLE = asset is not spendable -POLL_DOES_NOT_EXIST = POLL_DOES_NOT_EXIST +ACCOUNT_CANNOT_REWARD_SHARE = account cannot reward-share + +SELF_SHARE_EXISTS = self-share (reward-share) already exists -POLL_OPTION_DOES_NOT_EXIST = POLL_OPTION_DOES_NOT_EXIST +ACCOUNT_ALREADY_EXISTS = account already exists -PUBLIC_KEY_UNKNOWN = PUBLIC_KEY_UNKNOWN +INVALID_GROUP_BLOCK_DELAY = invalid group-approval block delay -SELF_SHARE_EXISTS = SELF_SHARE_EXISTS +INCORRECT_NONCE = incorrect PoW nonce -TIMESTAMP_TOO_NEW = TIMESTAMP_TOO_NEW +INVALID_TIMESTAMP_SIGNATURE = invalid timestamp signature -TIMESTAMP_TOO_OLD = TIMESTAMP_TOO_OLD +ADDRESS_BLOCKED = this address is blocked -TOO_MANY_UNCONFIRMED = TOO_MANY_UNCONFIRMED +NAME_BLOCKED = this name is blocked -TRANSACTION_ALREADY_CONFIRMED = TRANSACTION_ALREADY_CONFIRMED +ADDRESS_ABOVE_RATE_LIMIT = address reached specified rate limit -TRANSACTION_ALREADY_EXISTS = TRANSACTION_ALREADY_EXISTS +DUPLICATE_MESSAGE = address sent duplicate message -TRANSACTION_UNKNOWN = TRANSACTION_UNKNOWN +INVALID_BUT_OK = invalid but OK -TX_GROUP_ID_MISMATCH = TX_GROUP_ID_MISMATCH +NOT_YET_RELEASED = feature not yet released diff --git a/src/main/resources/i18n/TransactionValidity_fi.properties b/src/main/resources/i18n/TransactionValidity_fi.properties new file mode 100644 index 000000000..002ad5608 --- /dev/null +++ b/src/main/resources/i18n/TransactionValidity_fi.properties @@ -0,0 +1,193 @@ +ACCOUNT_ALREADY_EXISTS = tili on jo olemassa + +ACCOUNT_CANNOT_REWARD_SHARE = tili ei voi palkinto-jakaa + +ALREADY_GROUP_ADMIN = on jo ryhmän admin + +ALREADY_GROUP_MEMBER = on jo ryhmän jäsen + +ALREADY_VOTED_FOR_THAT_OPTION = on jo äänestänyt vaihtoehtoa + +ASSET_ALREADY_EXISTS = resurssi on jo olemassa + +ASSET_DOES_NOT_EXIST = resurssia ei ole olemassa + +ASSET_DOES_NOT_MATCH_AT = resurssi ei vastaa AT:n resurssia + +ASSET_NOT_SPENDABLE = resurssi ei ole kulutettavaa laatua + +AT_ALREADY_EXISTS = AT on jo olemassa + +AT_IS_FINISHED = AT on päättynyt + +AT_UNKNOWN = AT on tuntematon + +BANNED_FROM_GROUP = on evätty ryhmän jäsenyydestä + +BAN_EXISTS = eväys on jo olemassa + +BAN_UNKNOWN = tuntematon eväys + +BUYER_ALREADY_OWNER = ostaja on jo omistaja + +CLOCK_NOT_SYNCED = kello on synkronisoimatta + +DUPLICATE_OPTION = kahdennettu valinta + +GROUP_ALREADY_EXISTS = ryhmä on jo olemassa + +GROUP_APPROVAL_DECIDED = ryhmä-hyväksyminen jo päätetty + +GROUP_APPROVAL_NOT_REQUIRED = ryhmä-hyväksyminen tarpeeton + +GROUP_DOES_NOT_EXIST = ryhmää ei ole + +GROUP_ID_MISMATCH = ryhmän ID:n vastaavuusvirhe + +GROUP_OWNER_CANNOT_LEAVE = ryhmän omistaja ei voi jättää ryhmää + +HAVE_EQUALS_WANT = have-resurssi on sama kuin want-resurssi + +INCORRECT_NONCE = virheellinen PoW nonce + +INSUFFICIENT_FEE = riittämätön kulu + +INVALID_ADDRESS = kelvoton osoite + +INVALID_AMOUNT = kelvoton summa + +INVALID_ASSET_OWNER = kelvoton resurssin omistaja + +INVALID_AT_TRANSACTION = kelvoton AT-transaktio + +INVALID_AT_TYPE_LENGTH = kelvoton AT 'tyypin' pituus + +INVALID_CREATION_BYTES = kelvoton luodun tavumäärä + +INVALID_DATA_LENGTH = kelvoton datan pituus + +INVALID_DESCRIPTION_LENGTH = kelvoton kuvauksen pituus + +INVALID_GROUP_APPROVAL_THRESHOLD = kelvoton ryhmä-hyväksymisen alaraja + +INVALID_GROUP_BLOCK_DELAY = kelvoton ryhmä-hyväksymisen lohkon viive + +INVALID_GROUP_ID = kelvoton ryhmän ID + +INVALID_GROUP_OWNER = kelvoton ryhmän omistaja + +INVALID_LIFETIME = kelvoton elinaika + +INVALID_NAME_LENGTH = kelvoton nimen pituus + +INVALID_NAME_OWNER = kelvoton nimen omistaja + +INVALID_OPTIONS_COUNT = kelvoton valintojen lkm + +INVALID_OPTION_LENGTH = kelvoton valintojen pituus + +INVALID_ORDER_CREATOR = kelvoton tilauksen luoja + +INVALID_PAYMENTS_COUNT = kelvoton maksujen lkm + +INVALID_PUBLIC_KEY = kelvoton julkinen avain + +INVALID_QUANTITY = kelvoton määrä + +INVALID_REFERENCE = kelvoton viite + +INVALID_RETURN = kelvoton palautusarvo + +INVALID_REWARD_SHARE_PERCENT = kelvoton palkkiojaon prosenttiosuus + +INVALID_SELLER = kelvoton myyjä + +INVALID_TAGS_LENGTH = kelvoton 'tagin' pituus + +INVALID_TX_GROUP_ID = kelvoton transaktion ryhmä-ID + +INVALID_VALUE_LENGTH = kelvoton 'arvon' pituus + +INVITE_UNKNOWN = tuntematon ryhmän kutsu + +JOIN_REQUEST_EXISTS = ryhmään liittymispyyntö on jo olemassa + +MAXIMUM_REWARD_SHARES = tämän tilin suurin sallittu palkkiojaon lkm on saavutettu + +MISSING_CREATOR = luoja puuttuu + +MULTIPLE_NAMES_FORBIDDEN = yhdelle tilille sallitaan vain yksi rekisteröity nimi + +NAME_ALREADY_FOR_SALE = nimi on jo myynnissä + +NAME_ALREADY_REGISTERED = nimi on jo rekisteröity + +NAME_DOES_NOT_EXIST = nimeä ei ole + +NAME_NOT_FOR_SALE = nimi ei ole kaupan + +NAME_NOT_NORMALIZED = nimi ei ole Unicode 'normalisoitua' muotoa + +NEGATIVE_AMOUNT = kelvoton/negatiivinen summa + +NEGATIVE_FEE = kelvoton/negatiivinen kulu + +NEGATIVE_PRICE = kelvoton/negatiivinen hinta + +NOT_GROUP_ADMIN = tili ei ole ryhmän admin + +NOT_GROUP_MEMBER = tili ei ole ryhmän jäsen + +NOT_MINTING_ACCOUNT = tili ei voi lyödä rahaa + +NOT_YET_RELEASED = ominaisuutta ei ole vielä julkistettu + +NO_BALANCE = riittämätön saldo + +NO_BLOCKCHAIN_LOCK = solmun lohkoketju on juuri nyt varattuna + +NO_FLAG_PERMISSION = tilillä ei ole lupaa tuohon + +OK = OK + +ORDER_ALREADY_CLOSED = resurssin määräys kauppaan on jo suljettu + +ORDER_DOES_NOT_EXIST = resurssin määräystä kauppaan ei ole + +POLL_ALREADY_EXISTS = kysely on jo olemassa + +POLL_DOES_NOT_EXIST = kyselyä ei ole + +POLL_OPTION_DOES_NOT_EXIST = kyselyn tuota valintaa ei ole olemassa + +PUBLIC_KEY_UNKNOWN = tuntematon julkinen avain + +REWARD_SHARE_UNKNOWN = tuntematon palkkiojako + +SELF_SHARE_EXISTS = itse-jako (palkkiojako) on jo olemassa + +TIMESTAMP_TOO_NEW = aikaleima on liian tuore + +TIMESTAMP_TOO_OLD = aikaleima on liian vanha + +TOO_MANY_UNCONFIRMED = tilillä on liian monta vahvistamatonta transaktiota tekeillä + +TRANSACTION_ALREADY_CONFIRMED = transaktio on jo vahvistettu + +TRANSACTION_ALREADY_EXISTS = transaktio on jo olemassa + +TRANSACTION_UNKNOWN = tuntematon transaktio + +TX_GROUP_ID_MISMATCH = transaktion ryhmä-ID:n vastaavuusvirhe + +ADDRESS_BLOCKED = this address is blocked + +NAME_BLOCKED = this name is blocked + +ADDRESS_ABOVE_RATE_LIMIT = address reached specified rate limit + +DUPLICATE_MESSAGE = address sent duplicate message + +INVALID_TIMESTAMP_SIGNATURE = Invalid timestamp signature + +INVALID_BUT_OK = Invalid but OK diff --git a/src/main/resources/i18n/TransactionValidity_fr.properties b/src/main/resources/i18n/TransactionValidity_fr.properties new file mode 100644 index 000000000..6b43d4572 --- /dev/null +++ b/src/main/resources/i18n/TransactionValidity_fr.properties @@ -0,0 +1,151 @@ +OK = OK + +INVALID_ADDRESS = adresse invalide + +NEGATIVE_AMOUNT = montant invalide/négatif + +NEGATIVE_FEE = frais invalides/négatifs + +NO_BALANCE = solde insuffisant + +INVALID_REFERENCE = référence invalide + +INVALID_NAME_LENGTH = longueur de nom invalide + +INVALID_VALUE_LENGTH = longueur de 'valeur' invalide + +NAME_ALREADY_REGISTERED = le nom est déjà enregistré + +NAME_DOES_NOT_EXIST = le nom n'existe pas + +INVALID_NAME_OWNER = le nom du propriétaire est invalide + +NAME_ALREADY_FOR_SALE = le nom est déjà en vente + +NAME_NOT_FOR_SALE = le nom n'est pas à vendre + +BUYER_ALREADY_OWNER = l'acheteur est déjà le propriétaire + +INVALID_AMOUNT = montant invalide + +INVALID_SELLER = vendeur invalide + +NAME_NOT_NORMALIZED = le nom n'est pas sous la forme 'normalisée' Unicode + +INVALID_DESCRIPTION_LENGTH = longueur de description invalide + +INVALID_OPTIONS_COUNT = nombre d'options invalides + +INVALID_OPTION_LENGTH = longueur des options invalide + +DUPLICATE_OPTION = option dupliquée + +POLL_ALREADY_EXISTS = le scrutin existe déjà + +POLL_DOES_NOT_EXIST = le scrutin n'existe pas + +POLL_OPTION_DOES_NOT_EXIST = Ce choix de scrutin n'existe pas + +ALREADY_VOTED_FOR_THAT_OPTION = Vous avez déjà voté pour ce choix + +INVALID_DATA_LENGTH = longueur de données invalide + +INVALID_QUANTITY = quantité invalide + +ASSET_DOES_NOT_EXIST = l'actif n'existe pas + +INVALID_RETURN = retour invalide + +HAVE_EQUALS_WANT = l'actif désiré est le même que l'actif possédé + +ORDER_DOES_NOT_EXIST = l'ordre d'échange d'actifs n'existe pas + +INVALID_ORDER_CREATOR = créateur d'ordre invalide + +INVALID_PAYMENTS_COUNT = nombre de paiements invalides + +NEGATIVE_PRICE = prix invalide/négatif + +INVALID_CREATION_BYTES = octets de création invalides + +INVALID_TAGS_LENGTH = longueur de 'tags' invalide + +INVALID_AT_TYPE_LENGTH = longueur 'type' AT invalide + +INVALID_AT_TRANSACTION = transaction AT invalide + +INSUFFICIENT_FEE = frais insuffisant + +ASSET_DOES_NOT_MATCH_AT = l'actif ne correspond pas à l'actif d'AT +ASSET_ALREADY_EXISTS = l'actif existe déjà +MISSING_CREATOR = créateur manquant +TIMESTAMP_TOO_OLD = horodatage trop ancien +TIMESTAMP_TOO_NEW = horodatage trop récent +TOO_MANY_UNCONFIRMED = le compte a trop de transactions non confirmées en attente +GROUP_ALREADY_EXISTS = le groupe existe déjà +GROUP_DOES_NOT_EXIST = le groupe n'existe pas +INVALID_GROUP_OWNER = propriétaire de groupe invalide +ALREADY_GROUP_MEMBER = vous êtes déjà un(e) membre du groupe +GROUP_OWNER_CANNOT_LEAVE = le propriétaire du groupe ne peut pas quitter le groupe +NOT_GROUP_MEMBER = le compte n'est pas membre du groupe +ALREADY_GROUP_ADMIN = vous êtes déjà l'administrateur(trice) du groupe +NOT_GROUP_ADMIN = le compte n'est pas un administrateur du groupe +INVALID_LIFETIME = durée de vie invalide +INVITE_UNKNOWN = invitation de groupe inconnue +BAN_EXISTS = déjà banni +BAN_UNKNOWN = bannissement inconnu +BANNED_FROM_GROUP = banned from group +JOIN_REQUEST_EXISTS = la demande d'adhésion au groupe existe déjà +INVALID_GROUP_APPROVAL_THRESHOLD = seuil d'approbation de groupe non valide +GROUP_ID_MISMATCH = identifiant de groupe non-concorde +INVALID_GROUP_ID = identifiant de groupe invalide +TRANSACTION_UNKNOWN = transaction inconnue +TRANSACTION_ALREADY_CONFIRMED = la transaction a déjà été confirmée +INVALID_TX_GROUP_ID = identifiant du groupe de transactions invalide +TX_GROUP_ID_MISMATCH = l'identifiant du groupe de transaction ne correspond pas + +MULTIPLE_NAMES_FORBIDDEN = l'enregistrement de plusieurs noms par compte est interdit + +INVALID_ASSET_OWNER = propriétaire de l'actif invalide + +AT_IS_FINISHED = l'AT est fini + +NO_FLAG_PERMISSION = le compte n'a pas cette autorisation + +NOT_MINTING_ACCOUNT = le compte ne peut pas mint + +REWARD_SHARE_UNKNOWN = partage de récompense inconnu + +INVALID_REWARD_SHARE_PERCENT = pourcentage du partage de récompense invalide + +PUBLIC_KEY_UNKNOWN = clé publique inconnue + +INVALID_PUBLIC_KEY = clé publique invalide + +AT_UNKNOWN = AT inconnu + +AT_ALREADY_EXISTS = AT déjà existante + +GROUP_APPROVAL_NOT_REQUIRED = approbation de groupe non requise + +GROUP_APPROVAL_DECIDED = approbation de groupe déjà décidée + +MAXIMUM_REWARD_SHARES = déjà au nombre maximum de récompense pour ce compte + +TRANSACTION_ALREADY_EXISTS = la transaction existe déjà + +NO_BLOCKCHAIN_LOCK = nœud de la blockchain actuellement occupé +ORDER_ALREADY_CLOSED = l'ordre d'échange d'actifs est déjà fermé +CLOCK_NOT_SYNCED = horloge non synchronisée +ASSET_NOT_SPENDABLE = l'actif n'est pas dépensable +ACCOUNT_CANNOT_REWARD_SHARE = le compte ne peut pas récompenser +SELF_SHARE_EXISTS = l'auto-partage (récompense) existe déjà +ACCOUNT_ALREADY_EXISTS = Le compte existe déjà +INVALID_GROUP_BLOCK_DELAY = délai de blocage d'approbation de groupe invalide +INCORRECT_NONCE = PoW nonce incorrect +INVALID_TIMESTAMP_SIGNATURE = signature d'horodatage invalide +ADDRESS_IN_BLACKLIST = cette adresse est dans votre liste noire +ADDRESS_ABOVE_RATE_LIMIT = l'adresse a atteint la limite de débit spécifiée +DUPLICATE_MESSAGE = l'adresse a envoyé un message en double +INVALID_BUT_OK = invalide mais OK +NOT_YET_RELEASED = fonctionnalité pas encore publiée diff --git a/src/main/resources/i18n/TransactionValidity_hu.properties b/src/main/resources/i18n/TransactionValidity_hu.properties new file mode 100644 index 000000000..bb43e18ff --- /dev/null +++ b/src/main/resources/i18n/TransactionValidity_hu.properties @@ -0,0 +1,195 @@ +# Magyar myelvre forditotta: Szkíta (Scythian). 2021 Augusztus 7. + +OK = OK + +INVALID_ADDRESS = érvénytelen név vagy cím + +NEGATIVE_AMOUNT = negatív összeg + +NEGATIVE_FEE = érvénytelen/negatív tranzakciós díj + +NO_BALANCE = elégtelen egyenleg + +INVALID_REFERENCE = érvénytelen hivatkozás + +INVALID_NAME_LENGTH = érvénytelen névhossz + +INVALID_VALUE_LENGTH = érvénytelen értékhossz + +NAME_ALREADY_REGISTERED = ez a név már regisztrált + +NAME_DOES_NOT_EXIST = ez a név nem létezik + +INVALID_NAME_OWNER = érvénytelen név tulajdonos + +NAME_ALREADY_FOR_SALE = ez a név már eladó + +NAME_NOT_FOR_SALE = ez a név nem eladó + +BUYER_ALREADY_OWNER = ez a vevő már a tulajdonos + +INVALID_AMOUNT = érvénytelen összeg + +INVALID_SELLER = érvénytelen eladó + +NAME_NOT_NORMALIZED = ez a név nincs "normalizált" Unicode formátumban + +INVALID_DESCRIPTION_LENGTH = érvénytelen leíráshossz + +INVALID_OPTIONS_COUNT = invalid options count + +INVALID_OPTION_LENGTH = érvénytelen opciókszám + +DUPLICATE_OPTION = ez a lehetőség már létezik + +POLL_ALREADY_EXISTS = ez a szavazás már létezik + +POLL_DOES_NOT_EXIST = ez a szavazás nem létezik + +POLL_OPTION_DOES_NOT_EXIST = ez a szavazási lehetőség nem létezik + +ALREADY_VOTED_FOR_THAT_OPTION = erre a lehetőségre már szavaztál + +INVALID_DATA_LENGTH = érvénytelen adathossz + +INVALID_QUANTITY = érvénytelen mennyiség + +ASSET_DOES_NOT_EXIST = tőke nem létezik + +INVALID_RETURN = érvénytelen csere tőke + +HAVE_EQUALS_WANT = saját tőke egyenlő a csere tőkével + +ORDER_DOES_NOT_EXIST = tőke rendelés nem létezik + +INVALID_ORDER_CREATOR = érvénytelen rendelés létrehozó + +INVALID_PAYMENTS_COUNT = a kifizetések száma érvénytelen + +NEGATIVE_PRICE = érvénytelen/negatív ár + +INVALID_CREATION_BYTES = érvénytelen létrehozási bájtok + +INVALID_TAGS_LENGTH = érvénytelen cimkehossz + +INVALID_AT_TYPE_LENGTH = érvénytelen AT "típus" hossz + +INVALID_AT_TRANSACTION = érvénytelen AT tranzakció + +INSUFFICIENT_FEE = elégtelen díj + +ASSET_DOES_NOT_MATCH_AT = a tőke nem egyezik az AT tőkéjével + +ASSET_ALREADY_EXISTS = ez a tőke már létezik + +MISSING_CREATOR = hiányzó létrehozó + +TIMESTAMP_TOO_OLD = időbélyeg túl régi + +TIMESTAMP_TOO_NEW = időbélyeg túl korai + +TOO_MANY_UNCONFIRMED = ennek a fióknak túl sok meg nem erősített tranzakciója van folyamatban + +GROUP_ALREADY_EXISTS = ez a csoport már létezik + +GROUP_DOES_NOT_EXIST = ez a csoport nem létezik + +INVALID_GROUP_OWNER = érvénytelen csoport tulajdonos + +ALREADY_GROUP_MEMBER = már csoporttag + +GROUP_OWNER_CANNOT_LEAVE = a csoport tulajdonos nem tudja elhagyni a csoportot + +NOT_GROUP_MEMBER = ez a tag nem csoporttag + +ALREADY_GROUP_ADMIN = már csoport adminisztrátor + +NOT_GROUP_ADMIN = ez a tag nem csoport adminisztrátor + +INVALID_LIFETIME = érvénytelen élettartam + +INVITE_UNKNOWN = ismeretlen csoport meghívás + +BAN_EXISTS = már ki van tiltva + +BAN_UNKNOWN = kitiltás nem létezik + +BANNED_FROM_GROUP = ki van tiltva a csoportból + +JOIN_REQUEST_EXISTS = a csoporthoz való csatlakozási kérelem már megtöretént + +INVALID_GROUP_APPROVAL_THRESHOLD = érvénytelen jóváhagyási küszöbérték + +GROUP_ID_MISMATCH = csoportazonosító nem egyezik + +INVALID_GROUP_ID = csoportazonosító érvénytelen + +TRANSACTION_UNKNOWN = ismeretlen tranzakció + +TRANSACTION_ALREADY_CONFIRMED = ez a tranzakció már meg van erősítve + +INVALID_TX_GROUP_ID = a tranzakció csoportazonosítója érvénytelen + +TX_GROUP_ID_MISMATCH = a tranzakció csoportazonosítója nem egyezik + +MULTIPLE_NAMES_FORBIDDEN = fiókonként több név regisztrálása tilos + +INVALID_ASSET_OWNER = érvénytelen tőke tulajdonos + +AT_IS_FINISHED = az AT végzett + +NO_FLAG_PERMISSION = ez a fiók nem rendelkezik ezzel az engedéllyel + +NOT_MINTING_ACCOUNT = ez a fiók nem tud QORT-ot verni + +REWARD_SHARE_UNKNOWN = ez a jutalék-megosztás ismeretlen + +INVALID_REWARD_SHARE_PERCENT = ez a jutalék-megosztási arány érvénytelen + +PUBLIC_KEY_UNKNOWN = ismeretlen nyilvános kulcs + +INVALID_PUBLIC_KEY = érvénytelen nyilvános kulcs + +AT_UNKNOWN = az AT ismeretlen + +AT_ALREADY_EXISTS = az AT már létezik + +GROUP_APPROVAL_NOT_REQUIRED = csoport általi jóváhagyás nem szükséges + +GROUP_APPROVAL_DECIDED = csoport általi jóváhagyás el van döntve + +MAXIMUM_REWARD_SHARES = ez a fiókcím már elérte a maximális lehetséges jutalék-megosztási részesedést + +TRANSACTION_ALREADY_EXISTS = ez a tranzakció már létezik + +NO_BLOCKCHAIN_LOCK = csomópont blokklánca jelenleg elfoglalt + +ORDER_ALREADY_CLOSED = ez a tőke értékesítés már befejeződött + +CLOCK_NOT_SYNCED = az óra nincs szinkronizálva + +ASSET_NOT_SPENDABLE = ez a tőke nem értékesíthető + +ACCOUNT_CANNOT_REWARD_SHARE = ez a fiók nem vehet részt jutalék-megosztásban + +SELF_SHARE_EXISTS = önrészes jutalék-megosztás már létezik + +ACCOUNT_ALREADY_EXISTS = ez a fiók már létezik + +INVALID_GROUP_BLOCK_DELAY = invalid group-approval block delay + +INCORRECT_NONCE = helytelen Proof-of-Work Nonce + +INVALID_TIMESTAMP_SIGNATURE = érvénytelen időbélyeg aláírás + +ADDRESS_BLOCKED = this address is blocked + +NAME_BLOCKED = this name is blocked + +ADDRESS_ABOVE_RATE_LIMIT = ez a cím elérte a megengedett mérték korlátot + +DUPLICATE_MESSAGE = ez a cím duplikált üzenetet küldött + +INVALID_BUT_OK = érvénytelen de elfogadva + +NOT_YET_RELEASED = ez a funkció még nem került kiadásra diff --git a/src/main/resources/i18n/TransactionValidity_it.properties b/src/main/resources/i18n/TransactionValidity_it.properties new file mode 100644 index 000000000..762f08656 --- /dev/null +++ b/src/main/resources/i18n/TransactionValidity_it.properties @@ -0,0 +1,195 @@ +# Italian translation by Pabs 2021 + +ACCOUNT_ALREADY_EXISTS = l'account gia esiste + +ACCOUNT_CANNOT_REWARD_SHARE = l'account non può fare la condivisione di ricompensa + +ALREADY_GROUP_ADMIN = è già amministratore del gruppo + +ALREADY_GROUP_MEMBER = è già membro del gruppo + +ALREADY_VOTED_FOR_THAT_OPTION = già votato per questa opzione + +ASSET_ALREADY_EXISTS = risorsa già esistente + +ASSET_DOES_NOT_EXIST = risorsa non esistente + +ASSET_DOES_NOT_MATCH_AT = l'asset non corrisponde all'asset di AT + +ASSET_NOT_SPENDABLE = la risorsa non è spendibile + +AT_ALREADY_EXISTS = AT gia esiste + +AT_IS_FINISHED = AT ha finito + +AT_UNKNOWN = AT sconosciuto + +BANNED_FROM_GROUP = divietato dal gruppo + +BAN_EXISTS = il divieto esiste già + +BAN_UNKNOWN = divieto sconosciuto + +BUYER_ALREADY_OWNER = l'acquirente è già proprietario + +CLOCK_NOT_SYNCED = orologio non sincronizzato + +DUPLICATE_OPTION = opzione duplicata + +GROUP_ALREADY_EXISTS = gruppo già esistente + +GROUP_APPROVAL_DECIDED = approvazione di gruppo già decisa + +GROUP_APPROVAL_NOT_REQUIRED = approvazione di gruppo non richiesto + +GROUP_DOES_NOT_EXIST = gruppo non esiste + +GROUP_ID_MISMATCH = identificazione di gruppo non corrispondente + +GROUP_OWNER_CANNOT_LEAVE = il proprietario del gruppo non può lasciare il gruppo + +HAVE_EQUALS_WANT = la risorsa avere è uguale a la risorsa volere + +INCORRECT_NONCE = PoW nonce sbagliato + +INSUFFICIENT_FEE = tariffa insufficiente + +INVALID_ADDRESS = indirizzo non valido + +INVALID_AMOUNT = importo non valido + +INVALID_ASSET_OWNER = proprietario della risorsa non valido + +INVALID_AT_TRANSACTION = transazione AT non valida + +INVALID_AT_TYPE_LENGTH = lunghezza di "tipo" AT non valida + +INVALID_CREATION_BYTES = byte di creazione non validi + +INVALID_DATA_LENGTH = lunghezza di dati non valida + +INVALID_DESCRIPTION_LENGTH = lunghezza della descrizione non valida + +INVALID_GROUP_APPROVAL_THRESHOLD = soglia di approvazione del gruppo non valida + +INVALID_GROUP_BLOCK_DELAY = ritardo del blocco di approvazione del gruppo non valido + +INVALID_GROUP_ID = identificazione di gruppo non valida + +INVALID_GROUP_OWNER = proprietario di gruppo non valido + +INVALID_LIFETIME = durata della vita non valida + +INVALID_NAME_LENGTH = lunghezza del nome non valida + +INVALID_NAME_OWNER = proprietario del nome non valido + +INVALID_OPTIONS_COUNT = conteggio di opzioni non validi + +INVALID_OPTION_LENGTH = lunghezza di opzioni non valida + +INVALID_ORDER_CREATOR = creatore dell'ordine non valido + +INVALID_PAYMENTS_COUNT = conteggio pagamenti non validi + +INVALID_PUBLIC_KEY = chiave pubblica non valida + +INVALID_QUANTITY = quantità non valida + +INVALID_REFERENCE = riferimento non valido + +INVALID_RETURN = ritorno non valido + +INVALID_REWARD_SHARE_PERCENT = percentuale condivisione di ricompensa non valida + +INVALID_SELLER = venditore non valido + +INVALID_TAGS_LENGTH = lunghezza dei "tag" non valida + +INVALID_TX_GROUP_ID = identificazione di gruppo di transazioni non valida + +INVALID_VALUE_LENGTH = lunghezza "valore" non valida + +INVITE_UNKNOWN = invito di gruppo sconosciuto + +JOIN_REQUEST_EXISTS = la richiesta di iscrizione al gruppo già esiste + +MAXIMUM_REWARD_SHARES = numero massimo di condivisione di ricompensa raggiunto per l'account + +MISSING_CREATOR = creatore mancante + +MULTIPLE_NAMES_FORBIDDEN = è vietata la registrazione di multipli nomi per account + +NAME_ALREADY_FOR_SALE = nome già in vendita + +NAME_ALREADY_REGISTERED = nome già registrato + +NAME_DOES_NOT_EXIST = il nome non esiste + +NAME_NOT_FOR_SALE = il nome non è in vendita + +NAME_NOT_NORMALIZED = il nome non è in forma "normalizzata" Unicode + +NEGATIVE_AMOUNT = importo non valido / negativo + +NEGATIVE_FEE = tariffa non valida / negativa + +NEGATIVE_PRICE = prezzo non valido / negativo + +NOT_GROUP_ADMIN = l'account non è un amministratore di gruppo + +NOT_GROUP_MEMBER = l'account non è un membro del gruppo + +NOT_MINTING_ACCOUNT = l'account non può coniare + +NOT_YET_RELEASED = funzione non ancora rilasciata + +NO_BALANCE = equilibrio insufficiente + +NO_BLOCKCHAIN_LOCK = nodo di blockchain attualmente occupato + +NO_FLAG_PERMISSION = l'account non dispone di questa autorizzazione + +OK = OK + +ORDER_ALREADY_CLOSED = l'ordine di scambio di risorsa è già chiuso + +ORDER_DOES_NOT_EXIST = l'ordine di scambio di risorsa non esiste + +POLL_ALREADY_EXISTS = il sondaggio già esiste + +POLL_DOES_NOT_EXIST = il sondaggio non esiste + +POLL_OPTION_DOES_NOT_EXIST = le opzioni di sondaggio non esistono + +PUBLIC_KEY_UNKNOWN = chiave pubblica sconosciuta + +REWARD_SHARE_UNKNOWN = condivisione di ricompensa sconosciuta + +SELF_SHARE_EXISTS = condivisione di sé (condivisione di ricompensa) già esiste + +TIMESTAMP_TOO_NEW = timestamp troppo nuovo + +TIMESTAMP_TOO_OLD = timestamp troppo vecchio + +TOO_MANY_UNCONFIRMED = l'account ha troppe transazioni non confermate in sospeso + +TRANSACTION_ALREADY_CONFIRMED = la transazione è già confermata + +TRANSACTION_ALREADY_EXISTS = la transazione già esiste + +TRANSACTION_UNKNOWN = transazione sconosciuta + +TX_GROUP_ID_MISMATCH = identificazione di gruppo della transazione non corrisponde + +ADDRESS_BLOCKED = this address is blocked + +NAME_BLOCKED = this name is blocked + +ADDRESS_ABOVE_RATE_LIMIT = address reached specified rate limit + +DUPLICATE_MESSAGE = address sent duplicate message + +INVALID_TIMESTAMP_SIGNATURE = Invalid timestamp signature + +INVALID_BUT_OK = Invalid but OK diff --git a/src/main/resources/i18n/TransactionValidity_nl.properties b/src/main/resources/i18n/TransactionValidity_nl.properties new file mode 100644 index 000000000..726af0a96 --- /dev/null +++ b/src/main/resources/i18n/TransactionValidity_nl.properties @@ -0,0 +1,193 @@ +ACCOUNT_ALREADY_EXISTS = account bestaat al + +ACCOUNT_CANNOT_REWARD_SHARE = account kan geen beloningen delen + +ALREADY_GROUP_ADMIN = reeds groepsadministrator + +ALREADY_GROUP_MEMBER = reeds groepslid + +ALREADY_VOTED_FOR_THAT_OPTION = reeds gestemd voor die optie + +ASSET_ALREADY_EXISTS = asset bestaat al + +ASSET_DOES_NOT_EXIST = asset bestaat niet + +ASSET_DOES_NOT_MATCH_AT = asset matcht niet met de asset van de AT + +ASSET_NOT_SPENDABLE = asset is niet uitgeefbaar + +AT_ALREADY_EXISTS = AT bestaat al + +AT_IS_FINISHED = AT is afgelopen + +AT_UNKNOWN = AT onbekend + +BANNED_FROM_GROUP = verbannen uit groep + +BAN_EXISTS = ban bestaat al + +BAN_UNKNOWN = ban onbekend + +BUYER_ALREADY_OWNER = koper is al eigenaar + +CLOCK_NOT_SYNCED = klok is niet gesynchronizeerd + +DUPLICATE_OPTION = dubbele optie + +GROUP_ALREADY_EXISTS = groep bestaat reeds + +GROUP_APPROVAL_DECIDED = groepsgoedkeuring reeds afgewezen + +GROUP_APPROVAL_NOT_REQUIRED = groepsgoedkeuring niet vereist + +GROUP_DOES_NOT_EXIST = groep bestaat niet + +GROUP_ID_MISMATCH = ongeldige match met groep-ID + +GROUP_OWNER_CANNOT_LEAVE = groepseigenaar kan de groep niet verlaten + +HAVE_EQUALS_WANT = have-asset is gelijk aan want-asset + +INCORRECT_NONCE = incorrecte PoW nonce + +INSUFFICIENT_FEE = vergoeding te laag + +INVALID_ADDRESS = ongeldig adres + +INVALID_AMOUNT = ongeldige hoeveelheid + +INVALID_ASSET_OWNER = ongeldige asset-eigenaar + +INVALID_AT_TRANSACTION = ongeldige AT-transactie + +INVALID_AT_TYPE_LENGTH = ongeldige lengte voor AT 'type' + +INVALID_CREATION_BYTES = ongeldige creation bytes + +INVALID_DATA_LENGTH = ongeldige lengte voor data + +INVALID_DESCRIPTION_LENGTH = ongeldige lengte voor beschrijving + +INVALID_GROUP_APPROVAL_THRESHOLD = ongeldige drempelwaarde voor groepsgoedkeuring + +INVALID_GROUP_BLOCK_DELAY = ongeldige groepsgoedkeuring voor blokvertraging + +INVALID_GROUP_ID = ongeldige groep-ID + +INVALID_GROUP_OWNER = ongeldige groepseigenaar + +INVALID_LIFETIME = ongeldige levensduur + +INVALID_NAME_LENGTH = ongeldige lengte voor naam + +INVALID_NAME_OWNER = ongeldige naam voor eigenaar + +INVALID_OPTIONS_COUNT = ongeldige hoeveelheid opties + +INVALID_OPTION_LENGTH = ongeldige lengte voor opties + +INVALID_ORDER_CREATOR = ongeldige aanmaker voor order + +INVALID_PAYMENTS_COUNT = ongeldige hoeveelheid betalingen + +INVALID_PUBLIC_KEY = ongeldige public key + +INVALID_QUANTITY = ongeldige hoeveelheid + +INVALID_REFERENCE = ongeldige verwijzing + +INVALID_RETURN = ongeldige return + +INVALID_REWARD_SHARE_PERCENT = ongeldig percentage voor beloningsdeling + +INVALID_SELLER = ongeldige verkoper + +INVALID_TAGS_LENGTH = ongeldige lengte voor 'tags' + +INVALID_TX_GROUP_ID = ongeldige transactiegroep-ID + +INVALID_VALUE_LENGTH = ongeldige lengte voor 'waarde' + +INVITE_UNKNOWN = onbekende groepsuitnodiging + +JOIN_REQUEST_EXISTS = aanvraag om lid van groep te worden bestaat al + +MAXIMUM_REWARD_SHARES = limiet aan beloningsdelingen voor dit account is bereikt + +MISSING_CREATOR = ontbrekende aanmaker + +MULTIPLE_NAMES_FORBIDDEN = het registreren van meerdere namen op een account is niet toegestaan + +NAME_ALREADY_FOR_SALE = naam reeds te koop + +NAME_ALREADY_REGISTERED = naam reeds geregistreerd + +NAME_DOES_NOT_EXIST = naam bestaat niet + +NAME_NOT_FOR_SALE = naam is niet te koop + +NAME_NOT_NORMALIZED = naam is niet in 'genormalizeerde' Unicode-vorm + +NEGATIVE_AMOUNT = ongeldige/negatieve hoeveelheid + +NEGATIVE_FEE = ongeldige/negatieve vergoeding + +NEGATIVE_PRICE = ongeldige/negatieve prijs + +NOT_GROUP_ADMIN = account is geen groepsadministrator + +NOT_GROUP_MEMBER = account is geen groepslid + +NOT_MINTING_ACCOUNT = account kan niet munten + +NOT_YET_RELEASED = functie nog niet uitgebracht + +NO_BALANCE = onvoldoende balans + +NO_BLOCKCHAIN_LOCK = blockchain van node is momenteel bezig + +NO_FLAG_PERMISSION = account heeft hier geen toestemming voor + +OK = Oke + +ORDER_ALREADY_CLOSED = asset handelsorder is al gesloten + +ORDER_DOES_NOT_EXIST = asset handelsorder bestaat niet + +POLL_ALREADY_EXISTS = peiling bestaat al + +POLL_DOES_NOT_EXIST = peiling bestaat niet + +POLL_OPTION_DOES_NOT_EXIST = peilingsoptie bestaat niet + +PUBLIC_KEY_UNKNOWN = public key onbekend + +REWARD_SHARE_UNKNOWN = beloningsdeling onbekend + +SELF_SHARE_EXISTS = zelfdeling (beloningsdeling) bestaat reeds + +TIMESTAMP_TOO_NEW = tijdstempel te nieuw + +TIMESTAMP_TOO_OLD = tijdstempel te oud + +TOO_MANY_UNCONFIRMED = account heeft te veel onbevestigde transacties in afwachting + +TRANSACTION_ALREADY_CONFIRMED = transactie is reeds bevestigd + +TRANSACTION_ALREADY_EXISTS = transactie bestaat al + +TRANSACTION_UNKNOWN = transactie onbekend + +TX_GROUP_ID_MISMATCH = groep-ID van transactie matcht niet + +ADDRESS_BLOCKED = this address is blocked + +NAME_BLOCKED = this name is blocked + +ADDRESS_ABOVE_RATE_LIMIT = address reached specified rate limit + +DUPLICATE_MESSAGE = address sent duplicate message + +INVALID_TIMESTAMP_SIGNATURE = Invalid timestamp signature + +INVALID_BUT_OK = Invalid but OK diff --git a/src/main/resources/i18n/TransactionValidity_ru.properties b/src/main/resources/i18n/TransactionValidity_ru.properties new file mode 100644 index 000000000..86e9d37ac --- /dev/null +++ b/src/main/resources/i18n/TransactionValidity_ru.properties @@ -0,0 +1,187 @@ +ACCOUNT_ALREADY_EXISTS = аккаунт уже существует + +ACCOUNT_CANNOT_REWARD_SHARE = аккаунт не может делиться вознаграждением + +ALREADY_GROUP_ADMIN = уже администратор группы + +ALREADY_GROUP_MEMBER = уже член группы + +ALREADY_VOTED_FOR_THAT_OPTION = уже проголосовали за этот вариант + +ASSET_ALREADY_EXISTS = актив уже существует + +ASSET_DOES_NOT_EXIST = Актив не существует + +ASSET_DOES_NOT_MATCH_AT = актив не совпадает с АТ + +ASSET_NOT_SPENDABLE = актив не подлежит расходованию + +AT_ALREADY_EXISTS = AT уже существует + +AT_IS_FINISHED = AT в завершении + +AT_UNKNOWN = не известный АТ + +BANNED_FROM_GROUP = исключен из группы + +BAN_EXISTS = Бан + +BAN_UNKNOWN = не известный бан + +BUYER_ALREADY_OWNER = покупатель уже собственник + +CLOCK_NOT_SYNCED = часы не синхронизированы + +DUPLICATE_OPTION = дублировать вариант + +GROUP_ALREADY_EXISTS = группа уже существует + +GROUP_APPROVAL_DECIDED = гуппа одобрена + +GROUP_APPROVAL_NOT_REQUIRED = гупповое одобрение не требуется + +GROUP_DOES_NOT_EXIST = группа не существует + +GROUP_ID_MISMATCH = не соответствие идентификатора группы + +GROUP_OWNER_CANNOT_LEAVE = владелец группы не может уйти + +HAVE_EQUALS_WANT = иммеются равные желания + +INSUFFICIENT_FEE = недостаточная плата + +INVALID_ADDRESS = недействительный адрес + +INVALID_AMOUNT = недопустимая сумма + +INVALID_ASSET_OWNER = недействительный владелец актива + +INVALID_AT_TRANSACTION = недействительная АТ транзакция + +INVALID_AT_TYPE_LENGTH = недействительно для типа длины AT + +INVALID_CREATION_BYTES = недопустимые байты создания + +INVALID_DATA_LENGTH = недопустимая длина данных + +INVALID_DESCRIPTION_LENGTH = недопустимая длина описания + +INVALID_GROUP_APPROVAL_THRESHOLD = недопустимый порог утверждения группы + +INVALID_GROUP_ID = недопустимый идентификатор группы + +INVALID_GROUP_OWNER = недопу владелец группы + +INVALID_LIFETIME = недопу срок службы + +INVALID_NAME_LENGTH = недопустимая длина группы + +INVALID_NAME_OWNER = недопустимое имя владельца + +INVALID_OPTIONS_COUNT = неверное количество опций + +INVALID_OPTION_LENGTH = недопустимая длина опции + +INVALID_ORDER_CREATOR = недопустимый создатель заказа + +INVALID_PAYMENTS_COUNT = неверный подсчет платежей + +INVALID_PUBLIC_KEY = недействительный открытый ключ + +INVALID_QUANTITY = недопустимое количество + +INVALID_REFERENCE = неверная ссылка + +INVALID_RETURN = недопустимый возврат + +INVALID_REWARD_SHARE_PERCENT = недействительный процент награждения + +INVALID_SELLER = недействительный продавец + +INVALID_TAGS_LENGTH = недействительная длина тэгов + +INVALID_TX_GROUP_ID = недействительный идентификатор группы передачи + +INVALID_VALUE_LENGTH = недопустимое значение длины + +INVITE_UNKNOWN = приглашать неизветсных + +JOIN_REQUEST_EXISTS = запрос на присоединение существует + +MAXIMUM_REWARD_SHARES = максимальное вознаграждение + +MISSING_CREATOR = отсутствующий создатель + +MULTIPLE_NAMES_FORBIDDEN = несколько имен запрещено + +NAME_ALREADY_FOR_SALE = имя уже в продаже + +NAME_ALREADY_REGISTERED = имя уже зарегистрировано + +NAME_DOES_NOT_EXIST = имя не существует + +NAME_NOT_FOR_SALE = имя не продается + +NAME_NOT_LOWER_CASE = иммя не должно содержать строчный регистр + +NEGATIVE_AMOUNT = недостаточная сумма + +NEGATIVE_FEE = недостаточная комиссия + +NEGATIVE_PRICE = недостаточная стоимость + +NOT_GROUP_ADMIN = не администратор группы + +NOT_GROUP_MEMBER = не член группы + +NOT_MINTING_ACCOUNT = счет не чеканит + +NOT_YET_RELEASED = еще не выпущено + +NO_BALANCE = нет баланса + +NO_BLOCKCHAIN_LOCK = блокчейн узла в настоящее время занят + +NO_FLAG_PERMISSION = нет разрешения на флаг + +OK = OK + +ORDER_ALREADY_CLOSED = заказ закрыт + +ORDER_DOES_NOT_EXIST = заказа не существует + +POLL_ALREADY_EXISTS = опрос уже существует + +POLL_DOES_NOT_EXIST = опроса не существует + +POLL_OPTION_DOES_NOT_EXIST = вариантов ответа не существует + +PUBLIC_KEY_UNKNOWN = открытый ключ неизвестен + +SELF_SHARE_EXISTS = поделиться долей + +TIMESTAMP_TOO_NEW = новая метка времени + +TIMESTAMP_TOO_OLD = старая метка времени + +TOO_MANY_UNCONFIRMED = много не подтвержденных + +TRANSACTION_ALREADY_CONFIRMED = транзакция уже подтверждена + +TRANSACTION_ALREADY_EXISTS = транзакция существует + +TRANSACTION_UNKNOWN = неизвестная транзакция + +TX_GROUP_ID_MISMATCH = не соответствие идентификатора группы c хэш транзации + +ADDRESS_BLOCKED = this address is blocked + +NAME_BLOCKED = this name is blocked + +ADDRESS_ABOVE_RATE_LIMIT = address reached specified rate limit + +DUPLICATE_MESSAGE = address sent duplicate message + +INVALID_TIMESTAMP_SIGNATURE = Invalid timestamp signature + +INVALID_BUT_OK = Invalid but OK diff --git a/src/main/resources/images/Qlogo_512.png b/src/main/resources/images/Qlogo_512.png new file mode 100644 index 000000000..81508bb70 Binary files /dev/null and b/src/main/resources/images/Qlogo_512.png differ diff --git a/src/main/resources/images/icons/Qlogo_128.png b/src/main/resources/images/icons/Qlogo_128.png new file mode 100644 index 000000000..463bb5274 Binary files /dev/null and b/src/main/resources/images/icons/Qlogo_128.png differ diff --git a/src/main/resources/images/icons/icon128.png b/src/main/resources/images/icons/icon128.png deleted file mode 100644 index ddb869bd3..000000000 Binary files a/src/main/resources/images/icons/icon128.png and /dev/null differ diff --git a/src/main/resources/images/icons/icon32.png b/src/main/resources/images/icons/icon32.png deleted file mode 100644 index 43a37510c..000000000 Binary files a/src/main/resources/images/icons/icon32.png and /dev/null differ diff --git a/src/main/resources/images/icons/qortal_ui_tray_minting.png b/src/main/resources/images/icons/qortal_ui_tray_minting.png new file mode 100644 index 000000000..567e784ba Binary files /dev/null and b/src/main/resources/images/icons/qortal_ui_tray_minting.png differ diff --git a/src/main/resources/images/icons/qortal_ui_tray_synced.png b/src/main/resources/images/icons/qortal_ui_tray_synced.png new file mode 100644 index 000000000..f944bad97 Binary files /dev/null and b/src/main/resources/images/icons/qortal_ui_tray_synced.png differ diff --git a/src/main/resources/images/icons/qortal_ui_tray_syncing.png b/src/main/resources/images/icons/qortal_ui_tray_syncing.png new file mode 100644 index 000000000..82d39bbb4 Binary files /dev/null and b/src/main/resources/images/icons/qortal_ui_tray_syncing.png differ diff --git a/src/main/resources/images/icons/qortal_ui_tray_syncing_time-alt.png b/src/main/resources/images/icons/qortal_ui_tray_syncing_time-alt.png new file mode 100644 index 000000000..608be51e4 Binary files /dev/null and b/src/main/resources/images/icons/qortal_ui_tray_syncing_time-alt.png differ diff --git a/src/main/resources/images/splash.png b/src/main/resources/images/splash.png old mode 100755 new mode 100644 diff --git a/src/main/resources/loading/index.html b/src/main/resources/loading/index.html new file mode 100644 index 000000000..fc50c72da --- /dev/null +++ b/src/main/resources/loading/index.html @@ -0,0 +1,274 @@ + + + + + Loading... + + + + + + + + + + + + + + +

+
+

Loading

+

+ Files are being retrieved from the Qortal Data Network. + This page will refresh automatically when the content becomes available. +

+

Loading...

+

+
+
+ + + diff --git a/src/test/java/org/qortal/test/AccountBalanceTests.java b/src/test/java/org/qortal/test/AccountBalanceTests.java index 35a9fb172..cd2822ac6 100644 --- a/src/test/java/org/qortal/test/AccountBalanceTests.java +++ b/src/test/java/org/qortal/test/AccountBalanceTests.java @@ -171,4 +171,84 @@ public void testRepositorySpeed() throws DataException, SQLException { Common.useDefaultSettings(); } + /** Test batch set/delete of account balances */ + @Test + public void testBatchedBalanceChanges() throws DataException, SQLException { + Random random = new Random(); + int ai; + + try (final Repository repository = RepositoryManager.getRepository()) { + System.out.println("Creating random accounts..."); + + // Generate some random accounts + List accounts = new ArrayList<>(); + for (ai = 0; ai < 2000; ++ai) { + byte[] publicKey = new byte[32]; + random.nextBytes(publicKey); + + PublicKeyAccount account = new PublicKeyAccount(repository, publicKey); + accounts.add(account); + } + + List accountBalances = new ArrayList<>(); + + System.out.println("Setting random balances..."); + + // Fill with lots of random balances + for (ai = 0; ai < accounts.size(); ++ai) { + Account account = accounts.get(ai); + int assetId = random.nextInt(2); + // random zero, or non-zero, balance + long balance = random.nextBoolean() ? 0L : random.nextInt(100000); + + accountBalances.add(new AccountBalanceData(account.getAddress(), assetId, balance)); + } + + repository.getAccountRepository().setAssetBalances(accountBalances); + repository.saveChanges(); + + System.out.println("Setting new random balances..."); + + // Now flip zero-ness for first half of balances + for (ai = 0; ai < accountBalances.size() / 2; ++ai) { + AccountBalanceData accountBalanceData = accountBalances.get(ai); + + accountBalanceData.setBalance(accountBalanceData.getBalance() != 0 ? 0L : random.nextInt(100000)); + } + // ...and randomize the rest + for (/*use ai from before*/; ai < accountBalances.size(); ++ai) { + AccountBalanceData accountBalanceData = accountBalances.get(ai); + + accountBalanceData.setBalance(random.nextBoolean() ? 0L : random.nextInt(100000)); + } + + repository.getAccountRepository().setAssetBalances(accountBalances); + repository.saveChanges(); + + System.out.println("Modifying random balances..."); + + // Fill with lots of random balance changes + for (ai = 0; ai < accounts.size(); ++ai) { + Account account = accounts.get(ai); + int assetId = random.nextInt(2); + // random zero, or non-zero, balance + long balance = random.nextBoolean() ? 0L : random.nextInt(100000); + + accountBalances.add(new AccountBalanceData(account.getAddress(), assetId, balance)); + } + + repository.getAccountRepository().modifyAssetBalances(accountBalances); + repository.saveChanges(); + + System.out.println("Deleting all balances..."); + + // Now simply delete all balances + for (ai = 0; ai < accountBalances.size(); ++ai) + accountBalances.get(ai).setBalance(0L); + + repository.getAccountRepository().setAssetBalances(accountBalances); + repository.saveChanges(); + } + } + } \ No newline at end of file diff --git a/src/test/java/org/qortal/test/BlockArchiveTests.java b/src/test/java/org/qortal/test/BlockArchiveTests.java new file mode 100644 index 000000000..e2f2ed1c9 --- /dev/null +++ b/src/test/java/org/qortal/test/BlockArchiveTests.java @@ -0,0 +1,705 @@ +package org.qortal.test; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.BlockMinter; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.*; +import org.qortal.repository.hsqldb.HSQLDBDatabaseArchiving; +import org.qortal.repository.hsqldb.HSQLDBDatabasePruning; +import org.qortal.repository.hsqldb.HSQLDBRepository; +import org.qortal.settings.Settings; +import org.qortal.test.common.AtUtils; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.transform.TransformationException; +import org.qortal.utils.BlockArchiveUtils; +import org.qortal.utils.NTP; +import org.qortal.utils.Triple; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.SQLException; +import java.util.List; + +import static org.junit.Assert.*; + +public class BlockArchiveTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useSettings("test-settings-v2-block-archive.json"); + NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset()); + this.deleteArchiveDirectory(); + } + + @After + public void afterTest() throws DataException { + this.deleteArchiveDirectory(); + } + + + @Test + public void testWriter() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) { + BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share")); + } + + // 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(900, maximumArchiveHeight); + + // Write blocks 2-900 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(900 - 1, writer.getWrittenCount()); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + } + } + + @Test + public void testWriterAndReader() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) { + BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share")); + } + + // 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(900, maximumArchiveHeight); + + // Write blocks 2-900 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(900 - 1, writer.getWrittenCount()); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + + // Read block 2 from the archive + BlockArchiveReader reader = BlockArchiveReader.getInstance(); + Triple, List> block2Info = reader.fetchBlockAtHeight(2); + BlockData block2ArchiveData = block2Info.getA(); + + // Read block 2 from the repository + BlockData block2RepositoryData = repository.getBlockRepository().fromHeight(2); + + // Ensure the values match + assertEquals(block2ArchiveData.getHeight(), block2RepositoryData.getHeight()); + assertArrayEquals(block2ArchiveData.getSignature(), block2RepositoryData.getSignature()); + + // Test some values in the archive + assertEquals(1, block2ArchiveData.getOnlineAccountsCount()); + + // Read block 900 from the archive + Triple, List> block900Info = reader.fetchBlockAtHeight(900); + BlockData block900ArchiveData = block900Info.getA(); + + // Read block 900 from the repository + BlockData block900RepositoryData = repository.getBlockRepository().fromHeight(900); + + // Ensure the values match + assertEquals(block900ArchiveData.getHeight(), block900RepositoryData.getHeight()); + assertArrayEquals(block900ArchiveData.getSignature(), block900RepositoryData.getSignature()); + + // Test some values in the archive + assertEquals(1, block900ArchiveData.getOnlineAccountsCount()); + + } + } + + @Test + public void testArchivedAtStates() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) { + BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share")); + } + + // 9 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(10); + repository.getATRepository().setAtTrimHeight(10); + + // Check the max archive height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(9, maximumArchiveHeight); + + // Write blocks 2-9 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(9 - 1, writer.getWrittenCount()); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(9 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + + // Check blocks 3-9 + for (Integer testHeight = 2; testHeight <= 9; testHeight++) { + + // Read a block from the archive + BlockArchiveReader reader = BlockArchiveReader.getInstance(); + Triple, List> blockInfo = reader.fetchBlockAtHeight(testHeight); + BlockData archivedBlockData = blockInfo.getA(); + ATStateData archivedAtStateData = blockInfo.getC().isEmpty() ? null : blockInfo.getC().get(0); + List archivedTransactions = blockInfo.getB(); + + // Read the same block from the repository + BlockData repositoryBlockData = repository.getBlockRepository().fromHeight(testHeight); + ATStateData repositoryAtStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight); + + // Ensure the repository has full AT state data + assertNotNull(repositoryAtStateData.getStateHash()); + assertNotNull(repositoryAtStateData.getStateData()); + + // Check the archived AT state + if (testHeight == 2) { + // Block 2 won't have an AT state hash because it's initial (and has the DEPLOY_AT in the same block) + assertNull(archivedAtStateData); + + assertEquals(1, archivedTransactions.size()); + assertEquals(Transaction.TransactionType.DEPLOY_AT, archivedTransactions.get(0).getType()); + } + else { + // For blocks 3+, ensure the archive has the AT state data, but not the hashes + assertNotNull(archivedAtStateData.getStateHash()); + assertNull(archivedAtStateData.getStateData()); + + // They also shouldn't have any transactions + assertTrue(archivedTransactions.isEmpty()); + } + + // Also check the online accounts count and height + assertEquals(1, archivedBlockData.getOnlineAccountsCount()); + assertEquals(testHeight, archivedBlockData.getHeight()); + + // Ensure the values match + assertEquals(archivedBlockData.getHeight(), repositoryBlockData.getHeight()); + assertArrayEquals(archivedBlockData.getSignature(), repositoryBlockData.getSignature()); + assertEquals(archivedBlockData.getOnlineAccountsCount(), repositoryBlockData.getOnlineAccountsCount()); + assertArrayEquals(archivedBlockData.getMinterSignature(), repositoryBlockData.getMinterSignature()); + assertEquals(archivedBlockData.getATCount(), repositoryBlockData.getATCount()); + assertEquals(archivedBlockData.getOnlineAccountsCount(), repositoryBlockData.getOnlineAccountsCount()); + assertArrayEquals(archivedBlockData.getReference(), repositoryBlockData.getReference()); + assertEquals(archivedBlockData.getTimestamp(), repositoryBlockData.getTimestamp()); + assertEquals(archivedBlockData.getATFees(), repositoryBlockData.getATFees()); + assertEquals(archivedBlockData.getTotalFees(), repositoryBlockData.getTotalFees()); + assertEquals(archivedBlockData.getTransactionCount(), repositoryBlockData.getTransactionCount()); + assertArrayEquals(archivedBlockData.getTransactionsSignature(), repositoryBlockData.getTransactionsSignature()); + + if (testHeight != 2) { + assertArrayEquals(archivedAtStateData.getStateHash(), repositoryAtStateData.getStateHash()); + } + } + + // Check block 10 (unarchived) + BlockArchiveReader reader = BlockArchiveReader.getInstance(); + Triple, List> blockInfo = reader.fetchBlockAtHeight(10); + assertNull(blockInfo); + + } + + } + + @Test + public void testArchiveAndPrune() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) { + BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share")); + } + + // Assume 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(900, maximumArchiveHeight); + + // Write blocks 2-900 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(900 - 1, writer.getWrittenCount()); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(901); + repository.saveChanges(); + assertEquals(901, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + + // Ensure the SQL repository contains blocks 2 and 900... + assertNotNull(repository.getBlockRepository().fromHeight(2)); + assertNotNull(repository.getBlockRepository().fromHeight(900)); + + // Prune all the archived blocks + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 900); + assertEquals(900-1, numBlocksPruned); + repository.getBlockRepository().setBlockPruneHeight(901); + + // Prune the AT states for the archived blocks + repository.getATRepository().rebuildLatestAtStates(); + int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 900); + assertEquals(900-1, numATStatesPruned); + repository.getATRepository().setAtPruneHeight(901); + + // Now ensure the SQL repository is missing blocks 2 and 900... + assertNull(repository.getBlockRepository().fromHeight(2)); + assertNull(repository.getBlockRepository().fromHeight(900)); + + // ... but it's not missing blocks 1 and 901 (we don't prune the genesis block) + assertNotNull(repository.getBlockRepository().fromHeight(1)); + assertNotNull(repository.getBlockRepository().fromHeight(901)); + + // Validate the latest block height in the repository + assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + } + } + + @Test + public void testBulkArchiveAndPrune() throws DataException, SQLException { + try (final Repository repository = RepositoryManager.getRepository()) { + HSQLDBRepository hsqldb = (HSQLDBRepository) repository; + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) { + BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share")); + } + + // Assume 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(900, maximumArchiveHeight); + + // Check the current archive height + assertEquals(0, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Write blocks 2-900 to the archive (using bulk method) + int fileSizeTarget = 425000; // Pre-calculated size of 900 blocks + assertTrue(HSQLDBDatabaseArchiving.buildBlockArchive(repository, fileSizeTarget)); + + // Ensure the block archive height has increased + assertEquals(901, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the SQL repository contains blocks 2 and 900... + assertNotNull(repository.getBlockRepository().fromHeight(2)); + assertNotNull(repository.getBlockRepository().fromHeight(900)); + + // Check the current prune heights + assertEquals(0, repository.getBlockRepository().getBlockPruneHeight()); + assertEquals(0, repository.getATRepository().getAtPruneHeight()); + + // Prior to archiving or pruning, ensure blocks 2 to 1002 and their AT states are available in the db + for (int i=2; i<=1002; i++) { + assertNotNull(repository.getBlockRepository().fromHeight(i)); + List atStates = repository.getATRepository().getBlockATStatesAtHeight(i); + assertNotNull(atStates); + assertEquals(1, atStates.size()); + } + + // Prune all the archived blocks and AT states (using bulk method) + assertTrue(HSQLDBDatabasePruning.pruneBlocks(hsqldb)); + assertTrue(HSQLDBDatabasePruning.pruneATStates(hsqldb)); + + // Ensure the current prune heights have increased + assertEquals(901, repository.getBlockRepository().getBlockPruneHeight()); + assertEquals(901, repository.getATRepository().getAtPruneHeight()); + + // Now ensure the SQL repository is missing blocks 2 and 900... + assertNull(repository.getBlockRepository().fromHeight(2)); + assertNull(repository.getBlockRepository().fromHeight(900)); + + // ... but it's not missing blocks 1 and 901 (we don't prune the genesis block) + assertNotNull(repository.getBlockRepository().fromHeight(1)); + assertNotNull(repository.getBlockRepository().fromHeight(901)); + + // Validate the latest block height in the repository + assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Ensure blocks 2-900 are all available in the archive + for (int i=2; i<=900; i++) { + assertNotNull(repository.getBlockArchiveRepository().fromHeight(i)); + } + + // Ensure blocks 2-900 are NOT available in the db + for (int i=2; i<=900; i++) { + assertNull(repository.getBlockRepository().fromHeight(i)); + } + + // Ensure blocks 901 to 1002 and their AT states are available in the db + for (int i=901; i<=1002; i++) { + assertNotNull(repository.getBlockRepository().fromHeight(i)); + List atStates = repository.getATRepository().getBlockATStatesAtHeight(i); + assertNotNull(atStates); + assertEquals(1, atStates.size()); + } + + // Ensure blocks 901 to 1002 are not available in the archive + for (int i=901; i<=1002; i++) { + assertNull(repository.getBlockArchiveRepository().fromHeight(i)); + } + } + } + + @Test + public void testBulkArchiveAndPruneMultipleFiles() throws DataException, SQLException { + try (final Repository repository = RepositoryManager.getRepository()) { + HSQLDBRepository hsqldb = (HSQLDBRepository) repository; + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) { + BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share")); + } + + // Assume 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(900, maximumArchiveHeight); + + // Check the current archive height + assertEquals(0, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Write blocks 2-900 to the archive (using bulk method) + int fileSizeTarget = 42000; // Pre-calculated size of approx 90 blocks + assertTrue(HSQLDBDatabaseArchiving.buildBlockArchive(repository, fileSizeTarget)); + + // Ensure 10 archive files have been created + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive"); + assertEquals(10, new File(archivePath.toString()).list().length); + + // Check the files exist + assertTrue(Files.exists(Paths.get(archivePath.toString(), "2-90.dat"))); + assertTrue(Files.exists(Paths.get(archivePath.toString(), "91-179.dat"))); + assertTrue(Files.exists(Paths.get(archivePath.toString(), "180-268.dat"))); + assertTrue(Files.exists(Paths.get(archivePath.toString(), "269-357.dat"))); + assertTrue(Files.exists(Paths.get(archivePath.toString(), "358-446.dat"))); + assertTrue(Files.exists(Paths.get(archivePath.toString(), "447-535.dat"))); + assertTrue(Files.exists(Paths.get(archivePath.toString(), "536-624.dat"))); + assertTrue(Files.exists(Paths.get(archivePath.toString(), "625-713.dat"))); + assertTrue(Files.exists(Paths.get(archivePath.toString(), "714-802.dat"))); + assertTrue(Files.exists(Paths.get(archivePath.toString(), "803-891.dat"))); + + // Ensure the block archive height has increased + // It won't be as high as 901, because blocks 892-901 were too small to reach the file size + // target of the 11th file + assertEquals(892, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the SQL repository contains blocks 2 and 891... + assertNotNull(repository.getBlockRepository().fromHeight(2)); + assertNotNull(repository.getBlockRepository().fromHeight(891)); + + // Check the current prune heights + assertEquals(0, repository.getBlockRepository().getBlockPruneHeight()); + assertEquals(0, repository.getATRepository().getAtPruneHeight()); + + // Prior to archiving or pruning, ensure blocks 2 to 1002 and their AT states are available in the db + for (int i=2; i<=1002; i++) { + assertNotNull(repository.getBlockRepository().fromHeight(i)); + List atStates = repository.getATRepository().getBlockATStatesAtHeight(i); + assertNotNull(atStates); + assertEquals(1, atStates.size()); + } + + // Prune all the archived blocks and AT states (using bulk method) + assertTrue(HSQLDBDatabasePruning.pruneBlocks(hsqldb)); + assertTrue(HSQLDBDatabasePruning.pruneATStates(hsqldb)); + + // Ensure the current prune heights have increased + assertEquals(892, repository.getBlockRepository().getBlockPruneHeight()); + assertEquals(892, repository.getATRepository().getAtPruneHeight()); + + // Now ensure the SQL repository is missing blocks 2 and 891... + assertNull(repository.getBlockRepository().fromHeight(2)); + assertNull(repository.getBlockRepository().fromHeight(891)); + + // ... but it's not missing blocks 1 and 901 (we don't prune the genesis block) + assertNotNull(repository.getBlockRepository().fromHeight(1)); + assertNotNull(repository.getBlockRepository().fromHeight(892)); + + // Validate the latest block height in the repository + assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Ensure blocks 2-891 are all available in the archive + for (int i=2; i<=891; i++) { + assertNotNull(repository.getBlockArchiveRepository().fromHeight(i)); + } + + // Ensure blocks 2-891 are NOT available in the db + for (int i=2; i<=891; i++) { + assertNull(repository.getBlockRepository().fromHeight(i)); + } + + // Ensure blocks 892 to 1002 and their AT states are available in the db + for (int i=892; i<=1002; i++) { + assertNotNull(repository.getBlockRepository().fromHeight(i)); + List atStates = repository.getATRepository().getBlockATStatesAtHeight(i); + assertNotNull(atStates); + assertEquals(1, atStates.size()); + } + + // Ensure blocks 892 to 1002 are not available in the archive + for (int i=892; i<=1002; i++) { + assertNull(repository.getBlockArchiveRepository().fromHeight(i)); + } + } + } + + @Test + public void testTrimArchivePruneAndOrphan() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) { + BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share")); + } + + // Make sure that block 500 has full AT state data and data hash + List block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500); + ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + + // Trim the first 500 blocks + repository.getBlockRepository().trimOldOnlineAccountsSignatures(0, 500); + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(501); + repository.getATRepository().trimAtStates(0, 500, 1000); + repository.getATRepository().setAtTrimHeight(501); + + // Now block 500 should only have the AT state data hash + block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500); + atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500); + assertNotNull(atStatesData.getStateHash()); + assertNull(atStatesData.getStateData()); + + // ... but block 501 should have the full data + List block501AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(501); + atStatesData = repository.getATRepository().getATStateAtHeight(block501AtStatesData.get(0).getATAddress(), 501); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(500, maximumArchiveHeight); + + BlockData block3DataPreArchive = repository.getBlockRepository().fromHeight(3); + + // Write blocks 2-500 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(500 - 1, writer.getWrittenCount()); // -1 for the genesis block + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(500 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + + // Ensure the SQL repository contains blocks 2 and 500... + assertNotNull(repository.getBlockRepository().fromHeight(2)); + assertNotNull(repository.getBlockRepository().fromHeight(500)); + + // Prune all the archived blocks + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 500); + assertEquals(500-1, numBlocksPruned); + repository.getBlockRepository().setBlockPruneHeight(501); + + // Prune the AT states for the archived blocks + repository.getATRepository().rebuildLatestAtStates(); + int numATStatesPruned = repository.getATRepository().pruneAtStates(2, 500); + assertEquals(499, numATStatesPruned); + repository.getATRepository().setAtPruneHeight(501); + + // Now ensure the SQL repository is missing blocks 2 and 500... + assertNull(repository.getBlockRepository().fromHeight(2)); + assertNull(repository.getBlockRepository().fromHeight(500)); + + // ... but it's not missing blocks 1 and 501 (we don't prune the genesis block) + assertNotNull(repository.getBlockRepository().fromHeight(1)); + assertNotNull(repository.getBlockRepository().fromHeight(501)); + + // Validate the latest block height in the repository + assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Now orphan some unarchived blocks. + BlockUtils.orphanBlocks(repository, 500); + assertEquals(502, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // We're close to the lower limit of the SQL database now, so + // we need to import some blocks from the archive + BlockArchiveUtils.importFromArchive(401, 500, repository); + + // Ensure the SQL repository now contains block 401 but not 400... + assertNotNull(repository.getBlockRepository().fromHeight(401)); + assertNull(repository.getBlockRepository().fromHeight(400)); + + // Import the remaining 399 blocks + BlockArchiveUtils.importFromArchive(2, 400, repository); + + // Verify that block 3 matches the original + BlockData block3DataPostArchive = repository.getBlockRepository().fromHeight(3); + assertArrayEquals(block3DataPreArchive.getSignature(), block3DataPostArchive.getSignature()); + assertEquals(block3DataPreArchive.getHeight(), block3DataPostArchive.getHeight()); + + // Orphan 1 more block, which should be the last one that is possible to be orphaned + BlockUtils.orphanBlocks(repository, 1); + + // Orphan another block, which should fail + Exception exception = null; + try { + BlockUtils.orphanBlocks(repository, 1); + } catch (DataException e) { + exception = e; + } + + // Ensure that a DataException is thrown because there is no more AT states data available + assertNotNull(exception); + assertEquals(DataException.class, exception.getClass()); + + // FUTURE: we may be able to retain unique AT states when trimming, to avoid this exception + // and allow orphaning back through blocks with trimmed AT states. + + } + } + + + /** + * Many nodes are missing an ATStatesHeightIndex due to an earlier bug + * In these cases we disable archiving and pruning as this index is a + * very essential component in these processes. + */ + @Test + public void testMissingAtStatesHeightIndex() throws DataException, SQLException { + try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) { + + // Firstly check that we're able to prune or archive when the index exists + assertTrue(repository.getATRepository().hasAtStatesHeightIndex()); + assertTrue(RepositoryManager.canArchiveOrPrune()); + + // Delete the index + repository.prepareStatement("DROP INDEX ATSTATESHEIGHTINDEX").execute(); + + // Ensure check that we're unable to prune or archive when the index doesn't exist + assertFalse(repository.getATRepository().hasAtStatesHeightIndex()); + assertFalse(RepositoryManager.canArchiveOrPrune()); + } + } + + + private void deleteArchiveDirectory() { + // Delete archive directory if exists + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath(); + try { + FileUtils.deleteDirectory(archivePath.toFile()); + } catch (IOException e) { + + } + } + +} diff --git a/src/test/java/org/qortal/test/BlockTests.java b/src/test/java/org/qortal/test/BlockTests.java index 285e6d81b..d6fdac02d 100644 --- a/src/test/java/org/qortal/test/BlockTests.java +++ b/src/test/java/org/qortal/test/BlockTests.java @@ -1,8 +1,13 @@ package org.qortal.test; +import java.util.Arrays; +import java.util.Deque; +import java.util.LinkedList; import java.util.List; +import java.util.stream.Collectors; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; import org.qortal.block.Block; @@ -79,6 +84,7 @@ public void testGenesisBlockTransactions() throws DataException { } @Test + @Ignore(value = "Doesn't work, to be fixed later") public void testBlockSerialization() throws DataException, TransformationException { try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount signingAccount = Common.getTestAccount(repository, "alice"); @@ -133,4 +139,171 @@ public void testBlockSerialization() throws DataException, TransformationExcepti } } + @Test + public void testLatestBlockCacheWithLatestBlock() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + Deque latestBlockCache = buildLatestBlockCache(repository, 20); + + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + byte[] parentSignature = latestBlock.getSignature(); + + List childBlocks = findCachedChildBlocks(latestBlockCache, parentSignature); + + assertEquals(true, childBlocks.isEmpty()); + } + } + + @Test + public void testLatestBlockCacheWithPenultimateBlock() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + Deque latestBlockCache = buildLatestBlockCache(repository, 20); + + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + BlockData penultimateBlock = repository.getBlockRepository().fromHeight(latestBlock.getHeight() - 1); + byte[] parentSignature = penultimateBlock.getSignature(); + + List childBlocks = findCachedChildBlocks(latestBlockCache, parentSignature); + + assertEquals(false, childBlocks.isEmpty()); + assertEquals(1, childBlocks.size()); + + BlockData expectedBlock = latestBlock; + BlockData actualBlock = childBlocks.get(0); + assertArrayEquals(expectedBlock.getSignature(), actualBlock.getSignature()); + } + } + + @Test + public void testLatestBlockCacheWithMiddleBlock() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + Deque latestBlockCache = buildLatestBlockCache(repository, 20); + + int tipOffset = 5; + + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + BlockData parentBlock = repository.getBlockRepository().fromHeight(latestBlock.getHeight() - tipOffset); + byte[] parentSignature = parentBlock.getSignature(); + + List childBlocks = findCachedChildBlocks(latestBlockCache, parentSignature); + + assertEquals(false, childBlocks.isEmpty()); + assertEquals(tipOffset, childBlocks.size()); + + BlockData expectedFirstBlock = repository.getBlockRepository().fromHeight(parentBlock.getHeight() + 1); + BlockData actualFirstBlock = childBlocks.get(0); + assertArrayEquals(expectedFirstBlock.getSignature(), actualFirstBlock.getSignature()); + + BlockData expectedLastBlock = latestBlock; + BlockData actualLastBlock = childBlocks.get(childBlocks.size() - 1); + assertArrayEquals(expectedLastBlock.getSignature(), actualLastBlock.getSignature()); + } + } + + @Test + public void testLatestBlockCacheWithFirstBlock() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + Deque latestBlockCache = buildLatestBlockCache(repository, 20); + + int tipOffset = latestBlockCache.size(); + + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + BlockData parentBlock = repository.getBlockRepository().fromHeight(latestBlock.getHeight() - tipOffset); + byte[] parentSignature = parentBlock.getSignature(); + + List childBlocks = findCachedChildBlocks(latestBlockCache, parentSignature); + + assertEquals(false, childBlocks.isEmpty()); + assertEquals(tipOffset, childBlocks.size()); + + BlockData expectedFirstBlock = repository.getBlockRepository().fromHeight(parentBlock.getHeight() + 1); + BlockData actualFirstBlock = childBlocks.get(0); + assertArrayEquals(expectedFirstBlock.getSignature(), actualFirstBlock.getSignature()); + + BlockData expectedLastBlock = latestBlock; + BlockData actualLastBlock = childBlocks.get(childBlocks.size() - 1); + assertArrayEquals(expectedLastBlock.getSignature(), actualLastBlock.getSignature()); + } + } + + @Test + public void testLatestBlockCacheWithNoncachedBlock() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + Deque latestBlockCache = buildLatestBlockCache(repository, 20); + + int tipOffset = latestBlockCache.size() + 1; // outside of cache + + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + BlockData parentBlock = repository.getBlockRepository().fromHeight(latestBlock.getHeight() - tipOffset); + byte[] parentSignature = parentBlock.getSignature(); + + List childBlocks = findCachedChildBlocks(latestBlockCache, parentSignature); + + assertEquals(true, childBlocks.isEmpty()); + } + } + + private Deque buildLatestBlockCache(Repository repository, int count) throws DataException { + Deque latestBlockCache = new LinkedList<>(); + + // Mint some blocks + for (int h = 0; h < count; ++h) + latestBlockCache.addLast(BlockUtils.mintBlock(repository).getBlockData()); + + // Reduce cache down to latest 10 blocks + while (latestBlockCache.size() > 10) + latestBlockCache.removeFirst(); + + return latestBlockCache; + } + + private List findCachedChildBlocks(Deque latestBlockCache, byte[] parentSignature) { + return latestBlockCache.stream() + .dropWhile(cachedBlockData -> !Arrays.equals(cachedBlockData.getReference(), parentSignature)) + .collect(Collectors.toList()); + } + + @Test + public void testCommonBlockSearch() { + // Given a list of block summaries, trim all trailing summaries after common block + + // We'll represent known block summaries as a list of booleans, + // where the boolean value indicates whether peer's block is also in our repository. + + // Trivial case, single element array + assertCommonBlock(0, new boolean[] { true }); + + // Test odd and even array lengths + for (int arrayLength = 5; arrayLength <= 6; ++arrayLength) { + boolean[] testBlocks = new boolean[arrayLength]; + + // Test increasing amount of common blocks + for (int c = 1; c <= testBlocks.length; ++c) { + testBlocks[c - 1] = true; + + assertCommonBlock(c - 1, testBlocks); + } + } + } + + private void assertCommonBlock(int expectedIndex, boolean[] testBlocks) { + int commonBlockIndex = findCommonBlockIndex(testBlocks); + assertEquals(expectedIndex, commonBlockIndex); + } + + private int findCommonBlockIndex(boolean[] testBlocks) { + int low = 1; + int high = testBlocks.length - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + + if (testBlocks[mid]) + low = mid + 1; + else + high = mid - 1; + } + + return low - 1; + } + } diff --git a/src/test/java/org/qortal/test/BootstrapTests.java b/src/test/java/org/qortal/test/BootstrapTests.java new file mode 100644 index 000000000..6baa57acd --- /dev/null +++ b/src/test/java/org/qortal/test/BootstrapTests.java @@ -0,0 +1,248 @@ +package org.qortal.test; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.BlockMinter; +import org.qortal.controller.Controller; +import org.qortal.data.block.BlockData; +import org.qortal.repository.*; +import org.qortal.settings.Settings; +import org.qortal.test.common.AtUtils; +import org.qortal.test.common.Common; +import org.qortal.transform.TransformationException; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + +public class BootstrapTests extends Common { + + @Before + public void beforeTest() throws DataException, IOException { + Common.useSettingsAndDb(Common.testSettingsFilename, false); + NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset()); + this.deleteBootstraps(); + } + + @After + public void afterTest() throws DataException, IOException { + this.deleteBootstraps(); + this.deleteExportDirectory(); + } + + @Test + public void testCheckRepositoryState() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + this.buildDummyBlockchain(repository); + + Bootstrap bootstrap = new Bootstrap(repository); + assertTrue(bootstrap.checkRepositoryState()); + + } + } + + @Test + public void testValidateBlockchain() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + this.buildDummyBlockchain(repository); + + Bootstrap bootstrap = new Bootstrap(repository); + assertTrue(bootstrap.validateBlockchain()); + + } + } + + + @Test + public void testCreateAndImportBootstrap() throws DataException, InterruptedException, TransformationException, IOException { + + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive", "2-900.dat"); + BlockData block1000; + byte[] originalArchiveContents; + + try (final Repository repository = RepositoryManager.getRepository()) { + this.buildDummyBlockchain(repository); + + Bootstrap bootstrap = new Bootstrap(repository); + Path bootstrapPath = bootstrap.getBootstrapOutputPath(); + + // Ensure the compressed bootstrap doesn't exist + assertFalse(Files.exists(bootstrapPath)); + + // Create bootstrap + bootstrap.create(); + + // Ensure the compressed bootstrap exists + assertTrue(Files.exists(bootstrapPath)); + + // Ensure the original block archive file exists + assertTrue(Files.exists(archivePath)); + originalArchiveContents = Files.readAllBytes(archivePath); + + // Ensure block 1000 exists in the repository + block1000 = repository.getBlockRepository().fromHeight(1000); + assertNotNull(block1000); + + // Ensure we can retrieve block 10 from the archive + assertNotNull(repository.getBlockArchiveRepository().fromHeight(10)); + + // Now delete block 1000 + repository.getBlockRepository().delete(block1000); + assertNull(repository.getBlockRepository().fromHeight(1000)); + + // Overwrite the archive with dummy data, and verify it + try (PrintWriter out = new PrintWriter(archivePath.toFile())) { + out.println("testdata"); + } + String newline = System.getProperty("line.separator"); + assertEquals("testdata", Files.readString(archivePath).replace(newline, "")); + + // Ensure we can no longer retrieve block 10 from the archive + assertNull(repository.getBlockArchiveRepository().fromHeight(10)); + + // Import the bootstrap back in + bootstrap.importFromPath(bootstrapPath); + } + + // We need a new connection because we have switched to a new repository + try (final Repository repository = RepositoryManager.getRepository()) { + + // Ensure the block archive file exists + assertTrue(Files.exists(archivePath)); + + // and that its contents match the original + assertArrayEquals(originalArchiveContents, Files.readAllBytes(archivePath)); + + // Make sure that block 1000 exists again + BlockData newBlock1000 = repository.getBlockRepository().fromHeight(1000); + assertNotNull(newBlock1000); + + // and ensure that the signatures match + assertArrayEquals(block1000.getSignature(), newBlock1000.getSignature()); + + // Ensure we can retrieve block 10 from the archive + assertNotNull(repository.getBlockArchiveRepository().fromHeight(10)); + } + } + + + private void buildDummyBlockchain(Repository repository) throws DataException, InterruptedException, TransformationException, IOException { + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Assume 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + + // Write blocks 2-900 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(901); + + // Prune all the archived blocks + repository.getBlockRepository().pruneBlocks(0, 900); + repository.getBlockRepository().setBlockPruneHeight(901); + + // Prune the AT states for the archived blocks + repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().pruneAtStates(0, 900); + repository.getATRepository().setAtPruneHeight(901); + + // Refill cache, used by Controller.getMinimumLatestBlockTimestamp() and other methods + Controller.getInstance().refillLatestBlocksCache(); + + repository.saveChanges(); + } + + @Test + public void testGetRandomHost() { + String[] bootstrapHosts = Settings.getInstance().getBootstrapHosts(); + List uniqueHosts = new ArrayList<>(); + + for (int i=0; i<1000; i++) { + Bootstrap bootstrap = new Bootstrap(); + String randomHost = bootstrap.getRandomHost(); + assertNotNull(randomHost); + + if (!uniqueHosts.contains(randomHost)){ + uniqueHosts.add(randomHost); + } + } + + // Ensure we have more than one bootstrap host in the settings + assertTrue(Arrays.asList(bootstrapHosts).size() > 1); + + // Ensure that all have been given the opportunity to be used + assertEquals(uniqueHosts.size(), Arrays.asList(bootstrapHosts).size()); + } + + private void deleteBootstraps() throws IOException { + try { + Path archivePath = Paths.get(String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), "bootstrap-archive.7z")); + Files.delete(archivePath); + + Path sha256Path = Paths.get(String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), "bootstrap-archive.7z.sha256")); + Files.delete(sha256Path); + + } catch (NoSuchFileException e) { + // Nothing to delete + } + + try { + Path path = Paths.get(String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), "bootstrap-toponly.7z")); + Files.delete(path); + + } catch (NoSuchFileException e) { + // Nothing to delete + } + + try { + Path path = Paths.get(String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), "bootstrap-full.7z")); + Files.delete(path); + + } catch (NoSuchFileException e) { + // Nothing to delete + } + } + + private void deleteExportDirectory() { + // Delete archive directory if exists + Path archivePath = Paths.get(Settings.getInstance().getExportPath()); + try { + FileUtils.deleteDirectory(archivePath.toFile()); + } catch (IOException e) { + + } + } + +} diff --git a/src/test/java/org/qortal/test/ByteArrayTests.java b/src/test/java/org/qortal/test/ByteArrayTests.java index 32c692ef8..8fb6f1cfc 100644 --- a/src/test/java/org/qortal/test/ByteArrayTests.java +++ b/src/test/java/org/qortal/test/ByteArrayTests.java @@ -3,10 +3,12 @@ import static org.junit.Assert.*; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.TreeMap; import org.junit.Before; import org.junit.Test; @@ -28,15 +30,13 @@ public void createTestValues() { } } - private void fillMap(Map map) { + private static void fillMap(Map map) { for (byte[] testValue : testValues) map.put(new ByteArray(testValue), String.valueOf(map.size())); } - private byte[] dup(byte[] value) { - byte[] copiedValue = new byte[value.length]; - System.arraycopy(value, 0, copiedValue, 0, copiedValue.length); - return copiedValue; + private static byte[] dup(byte[] value) { + return Arrays.copyOf(value, value.length); } @Test @@ -92,7 +92,7 @@ public void testCompareBoxedWithPrimitive() { @Test @SuppressWarnings("unlikely-arg-type") - public void testMapContainsKey() { + public void testHashMapContainsKey() { Map testMap = new HashMap<>(); fillMap(testMap); @@ -105,8 +105,59 @@ public void testMapContainsKey() { assertTrue("boxed not equal to primitive", ba.equals(copiedValue)); - // This won't work because copiedValue.hashCode() will not match ba.hashCode() - assertFalse("Primitive shouldn't be found in map", testMap.containsKey(copiedValue)); + /* + * Unfortunately this doesn't work because HashMap::containsKey compares hashCodes first, + * followed by object references, and copiedValue.hashCode() will never match ba.hashCode(). + */ + assertFalse("Primitive shouldn't be found in HashMap", testMap.containsKey(copiedValue)); + } + + @Test + @SuppressWarnings("unlikely-arg-type") + public void testTreeMapContainsKey() { + Map testMap = new TreeMap<>(); + fillMap(testMap); + + // Create new ByteArray object with an existing value. + byte[] copiedValue = dup(testValues.get(3)); + ByteArray ba = new ByteArray(copiedValue); + + // Confirm object can be found in map + assertTrue("ByteArray not found in map", testMap.containsKey(ba)); + + assertTrue("boxed not equal to primitive", ba.equals(copiedValue)); + + /* + * Unfortunately this doesn't work because TreeMap::containsKey(x) wants to cast x to + * Comparable and byte[] does not fit + * so this throws a ClassCastException. + */ + try { + assertFalse("Primitive shouldn't be found in TreeMap", testMap.containsKey(copiedValue)); + fail(); + } catch (ClassCastException e) { + // Expected + } + } + + @Test + @SuppressWarnings("unlikely-arg-type") + public void testArrayListContains() { + // Create new ByteArray object with an existing value. + byte[] copiedValue = dup(testValues.get(3)); + ByteArray ba = new ByteArray(copiedValue); + + // Confirm object can be found in list + assertTrue("ByteArray not found in map", testValues.contains(ba)); + + assertTrue("boxed not equal to primitive", ba.equals(copiedValue)); + + /* + * Unfortunately this doesn't work because ArrayList::contains performs + * copiedValue.equals(x) for each x in testValues, and byte[].equals() + * simply compares object references, so will never match any ByteArray. + */ + assertFalse("Primitive shouldn't be found in ArrayList", testValues.contains(copiedValue)); } @Test @@ -116,8 +167,9 @@ public void debugBoxedVersusPrimitive() { byte[] copiedValue = dup(testValue); + System.out.println(String.format("Primitive hashCode: 0x%08x", testValue.hashCode())); System.out.println(String.format("Boxed hashCode: 0x%08x", ba1.hashCode())); - System.out.println(String.format("Primitive hashCode: 0x%08x", copiedValue.hashCode())); + System.out.println(String.format("Duplicated primitive hashCode: 0x%08x", copiedValue.hashCode())); } @Test diff --git a/src/test/java/org/qortal/test/ChainWeightTests.java b/src/test/java/org/qortal/test/ChainWeightTests.java index c580f30cc..e53c4c8e5 100644 --- a/src/test/java/org/qortal/test/ChainWeightTests.java +++ b/src/test/java/org/qortal/test/ChainWeightTests.java @@ -3,12 +3,15 @@ import static org.junit.Assert.*; import java.math.BigInteger; +import java.text.DecimalFormat; +import java.text.NumberFormat; import java.util.ArrayList; import java.util.List; import java.util.Random; import org.qortal.account.Account; import org.qortal.block.Block; +import org.qortal.block.BlockChain; import org.qortal.data.block.BlockSummaryData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -17,12 +20,21 @@ import org.qortal.test.common.TestAccount; import org.qortal.transform.Transformer; import org.qortal.transform.block.BlockTransformer; +import org.qortal.utils.NTP; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; public class ChainWeightTests extends Common { private static final Random RANDOM = new Random(); + private static final NumberFormat FORMATTER = new DecimalFormat("0.###E0"); + + @BeforeClass + public static void beforeClass() { + // We need this so that NTP.getTime() in Block.calcChainWeight() doesn't return null, causing NPE + NTP.setFixedOffset(0L); + } @Before public void beforeTest() throws DataException { @@ -89,7 +101,97 @@ public void testMoreAccountsBlock() throws DataException { } } - // Check that a longer chain beats a shorter chain + // Demonstrates that typical key distance ranges from roughly 1E75 to 1E77 + @Test + public void testKeyDistances() { + byte[] parentMinterKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + byte[] testKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + + for (int i = 0; i < 50; ++i) { + int parentHeight = RANDOM.nextInt(50000); + RANDOM.nextBytes(parentMinterKey); + RANDOM.nextBytes(testKey); + int minterLevel = RANDOM.nextInt(10) + 1; + + BigInteger keyDistance = Block.calcKeyDistance(parentHeight, parentMinterKey, testKey, minterLevel); + + System.out.println(String.format("Parent height: %d, minter level: %d, distance: %s", + parentHeight, + minterLevel, + FORMATTER.format(keyDistance))); + } + } + + // If typical key distance ranges from 1E75 to 1E77 + // then we want lots of online accounts to push a 1E75 distance + // towards 1E77 so that it competes with a 1E77 key that has hardly any online accounts + // 1E75 is approx. 2**249 so maybe that's a good value for Block.ACCOUNTS_COUNT_SHIFT + @Test + public void testMoreAccountsVersusKeyDistance() throws DataException { + BigInteger minimumBetterKeyDistance = BigInteger.TEN.pow(77); + BigInteger maximumWorseKeyDistance = BigInteger.TEN.pow(75); + + try (final Repository repository = RepositoryManager.getRepository()) { + final byte[] parentMinterKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + + TestAccount betterAccount = Common.getTestAccount(repository, "bob-reward-share"); + byte[] betterKey = betterAccount.getPublicKey(); + int betterMinterLevel = Account.getRewardShareEffectiveMintingLevel(repository, betterKey); + + TestAccount worseAccount = Common.getTestAccount(repository, "dilbert-reward-share"); + byte[] worseKey = worseAccount.getPublicKey(); + int worseMinterLevel = Account.getRewardShareEffectiveMintingLevel(repository, worseKey); + + // This is to check that the hard-coded keys ARE actually better/worse as expected, before moving on testing more online accounts + BigInteger betterKeyDistance; + BigInteger worseKeyDistance; + + int parentHeight = 0; + do { + ++parentHeight; + betterKeyDistance = Block.calcKeyDistance(parentHeight, parentMinterKey, betterKey, betterMinterLevel); + worseKeyDistance = Block.calcKeyDistance(parentHeight, parentMinterKey, worseKey, worseMinterLevel); + } while (betterKeyDistance.compareTo(minimumBetterKeyDistance) < 0 || worseKeyDistance.compareTo(maximumWorseKeyDistance) > 0); + + System.out.println(String.format("Parent height: %d, better key distance: %s, worse key distance: %s", + parentHeight, + FORMATTER.format(betterKeyDistance), + FORMATTER.format(worseKeyDistance))); + + for (int accountsCountShift = 244; accountsCountShift <= 256; accountsCountShift += 2) { + for (int worseAccountsCount = 1; worseAccountsCount <= 101; worseAccountsCount += 25) { + for (int betterAccountsCount = 1; betterAccountsCount <= 1001; betterAccountsCount += 250) { + BlockSummaryData worseKeyBlockSummary = new BlockSummaryData(parentHeight + 1, null, worseKey, betterAccountsCount); + BlockSummaryData betterKeyBlockSummary = new BlockSummaryData(parentHeight + 1, null, betterKey, worseAccountsCount); + + populateBlockSummaryMinterLevel(repository, worseKeyBlockSummary); + populateBlockSummaryMinterLevel(repository, betterKeyBlockSummary); + + BigInteger worseKeyBlockWeight = calcBlockWeight(parentHeight, parentMinterKey, worseKeyBlockSummary, accountsCountShift); + BigInteger betterKeyBlockWeight = calcBlockWeight(parentHeight, parentMinterKey, betterKeyBlockSummary, accountsCountShift); + + System.out.println(String.format("Shift: %d, worse key: %d accounts, %s diff; better key: %d accounts: %s diff; winner: %s", + accountsCountShift, + betterAccountsCount, // used with worseKey + FORMATTER.format(worseKeyBlockWeight), + worseAccountsCount, // used with betterKey + FORMATTER.format(betterKeyBlockWeight), + worseKeyBlockWeight.compareTo(betterKeyBlockWeight) > 0 ? "worse key/better accounts" : "better key/worse accounts" + )); + } + } + + System.out.println(); + } + } + } + + private static BigInteger calcBlockWeight(int parentHeight, byte[] parentBlockSignature, BlockSummaryData blockSummaryData, int accountsCountShift) { + BigInteger keyDistance = Block.calcKeyDistance(parentHeight, parentBlockSignature, blockSummaryData.getMinterPublicKey(), blockSummaryData.getMinterLevel()); + return BigInteger.valueOf(blockSummaryData.getOnlineAccountsCount()).shiftLeft(accountsCountShift).add(keyDistance); + } + + // Check that a longer chain has same weight as shorter/truncated chain @Test public void testLongerChain() throws DataException { try (final Repository repository = RepositoryManager.getRepository()) { @@ -97,16 +199,20 @@ public void testLongerChain() throws DataException { BlockSummaryData commonBlockSummary = genBlockSummary(repository, commonBlockHeight); byte[] commonBlockGeneratorKey = commonBlockSummary.getMinterPublicKey(); - List shorterChain = genBlockSummaries(repository, 3, commonBlockSummary); - List longerChain = genBlockSummaries(repository, shorterChain.size() + 1, commonBlockSummary); - - populateBlockSummariesMinterLevels(repository, shorterChain); + List longerChain = genBlockSummaries(repository, 6, commonBlockSummary); populateBlockSummariesMinterLevels(repository, longerChain); - BigInteger shorterChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockGeneratorKey, shorterChain); - BigInteger longerChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockGeneratorKey, longerChain); + List shorterChain = longerChain.subList(0, longerChain.size() / 2); + + final int mutualHeight = commonBlockHeight - 1 + Math.min(shorterChain.size(), longerChain.size()); + + BigInteger shorterChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockGeneratorKey, shorterChain, mutualHeight); + BigInteger longerChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockGeneratorKey, longerChain, mutualHeight); - assertEquals("longer chain should have greater weight", 1, longerChainWeight.compareTo(shorterChainWeight)); + if (NTP.getTime() >= BlockChain.getInstance().getCalcChainWeightTimestamp()) + assertEquals("longer chain should have same weight", 0, longerChainWeight.compareTo(shorterChainWeight)); + else + assertEquals("longer chain should have greater weight", 1, longerChainWeight.compareTo(shorterChainWeight)); } } diff --git a/src/test/java/org/qortal/test/CryptoTests.java b/src/test/java/org/qortal/test/CryptoTests.java index 0e294c631..6a0133d2b 100644 --- a/src/test/java/org/qortal/test/CryptoTests.java +++ b/src/test/java/org/qortal/test/CryptoTests.java @@ -3,15 +3,28 @@ import org.junit.Test; import org.qortal.account.PrivateKeyAccount; import org.qortal.block.BlockChain; +import org.qortal.crypto.AES; import org.qortal.crypto.BouncyCastle25519; import org.qortal.crypto.Crypto; import org.qortal.test.common.Common; +import org.qortal.utils.Base58; import static org.junit.Assert.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Random; -import org.bitcoinj.core.Base58; import org.bouncycastle.crypto.agreement.X25519Agreement; import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters; import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters; @@ -20,6 +33,11 @@ import com.google.common.hash.HashCode; +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; + public class CryptoTests extends Common { @Test @@ -40,6 +58,37 @@ public void testDoubleDigest() { assertArrayEquals(expected, digest); } + @Test + public void testFileDigest() throws IOException { + byte[] input = HashCode.fromString("00").asBytes(); + + Path tempPath = Files.createTempFile("", ".tmp"); + Files.write(tempPath, input, StandardOpenOption.CREATE); + + byte[] digest = Crypto.digest(tempPath.toFile()); + byte[] expected = HashCode.fromString("6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d").asBytes(); + + assertArrayEquals(expected, digest); + + Files.delete(tempPath); + } + + @Test + public void testFileDigestWithRandomData() throws IOException { + byte[] input = new byte[128]; + new Random().nextBytes(input); + + Path tempPath = Files.createTempFile("", ".tmp"); + Files.write(tempPath, input, StandardOpenOption.CREATE); + + byte[] fileDigest = Crypto.digest(tempPath.toFile()); + byte[] memoryDigest = Crypto.digest(input); + + assertArrayEquals(fileDigest, memoryDigest); + + Files.delete(tempPath); + } + @Test public void testPublicKeyToAddress() { byte[] publicKey = HashCode.fromString("775ada64a48a30b3bfc4f1db16bca512d4088704975a62bde78781ce0cba90d6").asBytes(); @@ -255,4 +304,68 @@ public void testProxyKeys() { assertEquals(expectedProxyPrivateKey, Base58.encode(proxyPrivateKey)); } + + @Test + public void testAESFileEncryption() throws NoSuchAlgorithmException, IOException, IllegalBlockSizeException, + InvalidKeyException, BadPaddingException, InvalidAlgorithmParameterException, NoSuchPaddingException { + + // Create temporary directory and file paths + java.nio.file.Path tempDir = Files.createTempDirectory("qortal-tests"); + String inputFilePath = tempDir.toString() + File.separator + "inputFile"; + String outputFilePath = tempDir.toString() + File.separator + "outputFile"; + String decryptedFilePath = tempDir.toString() + File.separator + "decryptedFile"; + String reencryptedFilePath = tempDir.toString() + File.separator + "reencryptedFile"; + + // Generate some dummy data + byte[] randomBytes = new byte[1024]; + new Random().nextBytes(randomBytes); + + // Write it to the input file + FileOutputStream outputStream = new FileOutputStream(inputFilePath); + outputStream.write(randomBytes); + + // Make sure only the input file exists + assertTrue(Files.exists(Paths.get(inputFilePath))); + assertFalse(Files.exists(Paths.get(outputFilePath))); + + // Encrypt + SecretKey aesKey = AES.generateKey(256); + AES.encryptFile("AES", aesKey, inputFilePath, outputFilePath); + assertTrue(Files.exists(Paths.get(outputFilePath))); + byte[] encryptedBytes = Files.readAllBytes(Paths.get(outputFilePath)); + + // Delete the input file + Files.delete(Paths.get(inputFilePath)); + assertFalse(Files.exists(Paths.get(inputFilePath))); + + // Decrypt + String encryptedFilePath = outputFilePath; + assertFalse(Files.exists(Paths.get(decryptedFilePath))); + AES.decryptFile("AES", aesKey, encryptedFilePath, decryptedFilePath); + assertTrue(Files.exists(Paths.get(decryptedFilePath))); + + // Delete the output file + Files.delete(Paths.get(outputFilePath)); + assertFalse(Files.exists(Paths.get(outputFilePath))); + + // Check that the decrypted file contents matches the original data + byte[] decryptedBytes = Files.readAllBytes(Paths.get(decryptedFilePath)); + assertTrue(Arrays.equals(decryptedBytes, randomBytes)); + assertEquals(1024, decryptedBytes.length); + + // Write the original data back to the input file + outputStream = new FileOutputStream(inputFilePath); + outputStream.write(randomBytes); + + // Now encrypt the data one more time using the same key + // This is to ensure the initialization vector produces a different result + AES.encryptFile("AES", aesKey, inputFilePath, reencryptedFilePath); + assertTrue(Files.exists(Paths.get(reencryptedFilePath))); + + // Make sure the ciphertexts do not match + byte[] reencryptedBytes = Files.readAllBytes(Paths.get(reencryptedFilePath)); + assertFalse(Arrays.equals(encryptedBytes, reencryptedBytes)); + + } + } diff --git a/src/test/java/org/qortal/test/GuiTests.java b/src/test/java/org/qortal/test/GuiTests.java index 0a3520032..0754d33b4 100644 --- a/src/test/java/org/qortal/test/GuiTests.java +++ b/src/test/java/org/qortal/test/GuiTests.java @@ -2,10 +2,12 @@ import java.awt.TrayIcon.MessageType; +import org.junit.Ignore; import org.junit.Test; import org.qortal.gui.SplashFrame; import org.qortal.gui.SysTray; +@Ignore public class GuiTests { @Test diff --git a/src/test/java/org/qortal/test/ImportExportTests.java b/src/test/java/org/qortal/test/ImportExportTests.java new file mode 100644 index 000000000..2306d4845 --- /dev/null +++ b/src/test/java/org/qortal/test/ImportExportTests.java @@ -0,0 +1,454 @@ +package org.qortal.test; + +import org.apache.commons.io.FileUtils; +import org.bitcoinj.core.Address; +import org.bitcoinj.core.AddressFormatException; +import org.bitcoinj.core.ECKey; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PublicKeyAccount; +import org.qortal.controller.tradebot.LitecoinACCTv1TradeBot; +import org.qortal.controller.tradebot.TradeBot; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crypto.Crypto; +import org.qortal.data.account.MintingAccountData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBImportExport; +import org.qortal.settings.Settings; +import org.qortal.test.common.Common; +import org.qortal.utils.NTP; +import org.qortal.utils.Triple; + +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + +public class ImportExportTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + this.deleteExportDirectory(); + } + + @After + public void afterTest() throws DataException { + this.deleteExportDirectory(); + } + + + @Test + public void testExportAndImportTradeBotStates() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Ensure no trade bots exist + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Create some trade bots + List tradeBots = new ArrayList<>(); + for (int i=0; i<10; i++) { + TradeBotData tradeBotData = this.createTradeBotData(repository); + repository.getCrossChainRepository().save(tradeBotData); + tradeBots.add(tradeBotData); + } + + // Ensure they have been added + assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // Export them + HSQLDBImportExport.backupTradeBotStates(repository, null); + + // Delete them from the repository + for (TradeBotData tradeBotData : tradeBots) { + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + } + + // Ensure they have been deleted + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Import them + Path exportPath = HSQLDBImportExport.getExportDirectory(false); + Path filePath = Paths.get(exportPath.toString(), "TradeBotStates.json"); + HSQLDBImportExport.importDataFromFile(filePath.toString(), repository); + + // Ensure they have been imported + assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // Ensure all the data matches + for (TradeBotData tradeBotData : tradeBots) { + byte[] tradePrivateKey = tradeBotData.getTradePrivateKey(); + TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + assertNotNull(repositoryTradeBotData); + assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString()); + } + + repository.saveChanges(); + } + } + + @Test + public void testExportAndImportCurrentTradeBotStates() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Ensure no trade bots exist + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Create some trade bots + List tradeBots = new ArrayList<>(); + for (int i=0; i<10; i++) { + TradeBotData tradeBotData = this.createTradeBotData(repository); + repository.getCrossChainRepository().save(tradeBotData); + tradeBots.add(tradeBotData); + } + + // Ensure they have been added + assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // Export them + HSQLDBImportExport.backupTradeBotStates(repository, null); + + // Delete them from the repository + for (TradeBotData tradeBotData : tradeBots) { + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + } + + // Ensure they have been deleted + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Add some more trade bots + List additionalTradeBots = new ArrayList<>(); + for (int i=0; i<5; i++) { + TradeBotData tradeBotData = this.createTradeBotData(repository); + repository.getCrossChainRepository().save(tradeBotData); + additionalTradeBots.add(tradeBotData); + } + + // Export again + HSQLDBImportExport.backupTradeBotStates(repository, null); + + // Import current states only + Path exportPath = HSQLDBImportExport.getExportDirectory(false); + Path filePath = Paths.get(exportPath.toString(), "TradeBotStates.json"); + HSQLDBImportExport.importDataFromFile(filePath.toString(), repository); + + // Ensure they have been imported + assertEquals(5, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // Ensure that only the additional trade bots have been imported and that the data matches + for (TradeBotData tradeBotData : additionalTradeBots) { + byte[] tradePrivateKey = tradeBotData.getTradePrivateKey(); + TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + assertNotNull(repositoryTradeBotData); + assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString()); + } + + // None of the original trade bots should exist in the repository + for (TradeBotData tradeBotData : tradeBots) { + byte[] tradePrivateKey = tradeBotData.getTradePrivateKey(); + TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + assertNull(repositoryTradeBotData); + } + + repository.saveChanges(); + } + } + + @Test + public void testExportAndImportAllTradeBotStates() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Ensure no trade bots exist + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Create some trade bots + List tradeBots = new ArrayList<>(); + for (int i=0; i<10; i++) { + TradeBotData tradeBotData = this.createTradeBotData(repository); + repository.getCrossChainRepository().save(tradeBotData); + tradeBots.add(tradeBotData); + } + + // Ensure they have been added + assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // Export them + HSQLDBImportExport.backupTradeBotStates(repository, null); + + // Delete them from the repository + for (TradeBotData tradeBotData : tradeBots) { + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + } + + // Ensure they have been deleted + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Add some more trade bots + List additionalTradeBots = new ArrayList<>(); + for (int i=0; i<5; i++) { + TradeBotData tradeBotData = this.createTradeBotData(repository); + repository.getCrossChainRepository().save(tradeBotData); + additionalTradeBots.add(tradeBotData); + } + + // Export again + HSQLDBImportExport.backupTradeBotStates(repository, null); + + // Import all states from the archive + Path exportPath = HSQLDBImportExport.getExportDirectory(false); + Path filePath = Paths.get(exportPath.toString(), "TradeBotStatesArchive.json"); + HSQLDBImportExport.importDataFromFile(filePath.toString(), repository); + + // Ensure they have been imported + assertEquals(15, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // Ensure that all known trade bots have been imported and that the data matches + tradeBots.addAll(additionalTradeBots); + + for (TradeBotData tradeBotData : tradeBots) { + byte[] tradePrivateKey = tradeBotData.getTradePrivateKey(); + TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + assertNotNull(repositoryTradeBotData); + assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString()); + } + + repository.saveChanges(); + } + } + + @Test + public void testExportAndImportLegacyTradeBotStates() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Create some trade bots, but don't save them in the repository + List tradeBots = new ArrayList<>(); + for (int i=0; i<10; i++) { + TradeBotData tradeBotData = this.createTradeBotData(repository); + tradeBots.add(tradeBotData); + } + + // Create a legacy format TradeBotStates.json backup file + this.exportLegacyTradeBotStatesJson(tradeBots); + + // Ensure no trade bots exist in repository + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Import the legacy format file + Path exportPath = HSQLDBImportExport.getExportDirectory(false); + Path filePath = Paths.get(exportPath.toString(), "TradeBotStates.json"); + HSQLDBImportExport.importDataFromFile(filePath.toString(), repository); + + // Ensure they have been imported + assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size()); + + for (TradeBotData tradeBotData : tradeBots) { + byte[] tradePrivateKey = tradeBotData.getTradePrivateKey(); + TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + assertNotNull(repositoryTradeBotData); + assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString()); + } + + repository.saveChanges(); + } + } + + @Test + public void testArchiveTradeBotStateOnTradeFailure() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Create a trade bot and save it in the repository + TradeBotData tradeBotData = this.createTradeBotData(repository); + + // Ensure it doesn't exist in the repository + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Export trade bot states, passing in the newly created trade bot as an additional parameter + // This is needed because it hasn't been saved to the db yet + HSQLDBImportExport.backupTradeBotStates(repository, Arrays.asList(tradeBotData)); + + // Ensure it is still not present in the repository + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Export all local node data again, but this time without including the trade bot data + // This simulates the behaviour of a node shutdown + repository.exportNodeLocalData(); + + // The TradeBotStates.json file should contain no entries + Path backupDirectory = HSQLDBImportExport.getExportDirectory(false); + Path tradeBotStatesBackup = Paths.get(backupDirectory.toString(), "TradeBotStates.json"); + assertTrue(Files.exists(tradeBotStatesBackup)); + String jsonString = new String(Files.readAllBytes(tradeBotStatesBackup)); + Triple parsedJSON = HSQLDBImportExport.parseJSONString(jsonString); + JSONArray tradeBotDataJson = parsedJSON.getC(); + assertTrue(tradeBotDataJson.isEmpty()); + + // .. but the TradeBotStatesArchive.json should contain the trade bot data + Path tradeBotStatesArchiveBackup = Paths.get(backupDirectory.toString(), "TradeBotStatesArchive.json"); + assertTrue(Files.exists(tradeBotStatesArchiveBackup)); + jsonString = new String(Files.readAllBytes(tradeBotStatesArchiveBackup)); + parsedJSON = HSQLDBImportExport.parseJSONString(jsonString); + JSONObject tradeBotDataJsonObject = (JSONObject) parsedJSON.getC().get(0); + assertEquals(tradeBotData.toJson().toString(), tradeBotDataJsonObject.toString()); + + // Now try importing local data (to simulate a node startup) + String exportPath = Settings.getInstance().getExportPath(); + Path importPath = Paths.get(exportPath, "TradeBotStates.json"); + repository.importDataFromFile(importPath.toString()); + + // The trade should be missing since it's not present in TradeBotStates.json + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // The user now imports TradeBotStatesArchive.json + Path archiveImportPath = Paths.get(exportPath, "TradeBotStatesArchive.json"); + repository.importDataFromFile(archiveImportPath.toString()); + + // The trade should be present in the database + assertEquals(1, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // The trade bot data in the repository should match the one that was originally created + byte[] tradePrivateKey = tradeBotData.getTradePrivateKey(); + TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + assertNotNull(repositoryTradeBotData); + assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString()); + } + } + + @Test + public void testExportAndImportMintingAccountData() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Ensure no minting accounts exist + assertTrue(repository.getAccountRepository().getMintingAccounts().isEmpty()); + + // Create some minting accounts + List mintingAccounts = new ArrayList<>(); + for (int i=0; i<10; i++) { + MintingAccountData mintingAccountData = this.createMintingAccountData(); + repository.getAccountRepository().save(mintingAccountData); + mintingAccounts.add(mintingAccountData); + } + + // Ensure they have been added + assertEquals(10, repository.getAccountRepository().getMintingAccounts().size()); + + // Export them + HSQLDBImportExport.backupMintingAccounts(repository); + + // Delete them from the repository + for (MintingAccountData mintingAccountData : mintingAccounts) { + repository.getAccountRepository().delete(mintingAccountData.getPrivateKey()); + } + + // Ensure they have been deleted + assertTrue(repository.getAccountRepository().getMintingAccounts().isEmpty()); + + // Import them + Path exportPath = HSQLDBImportExport.getExportDirectory(false); + Path filePath = Paths.get(exportPath.toString(), "MintingAccounts.json"); + HSQLDBImportExport.importDataFromFile(filePath.toString(), repository); + + // Ensure they have been imported + assertEquals(10, repository.getAccountRepository().getMintingAccounts().size()); + + // Ensure all the data matches + for (MintingAccountData mintingAccountData : mintingAccounts) { + byte[] privateKey = mintingAccountData.getPrivateKey(); + MintingAccountData repositoryMintingAccountData = repository.getAccountRepository().getMintingAccount(privateKey); + assertNotNull(repositoryMintingAccountData); + assertEquals(mintingAccountData.toJson().toString(), repositoryMintingAccountData.toJson().toString()); + } + + repository.saveChanges(); + } + } + + + private TradeBotData createTradeBotData(Repository repository) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + + String receivingAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + + // Convert Litecoin receiving address into public key hash (we only support P2PKH at this time) + Address litecoinReceivingAddress; + try { + litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), receivingAddress); + } catch (AddressFormatException e) { + throw new DataException("Unsupported Litecoin receiving address: " + receivingAddress); + } + + byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash(); + + byte[] creatorPublicKey = new byte[32]; + PublicKeyAccount creator = new PublicKeyAccount(repository, creatorPublicKey); + + long timestamp = NTP.getTime(); + String atAddress = "AT_ADDRESS"; + long foreignAmount = 1234; + long qortAmount= 5678; + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv1.NAME, + LitecoinACCTv1TradeBot.State.BOB_WAITING_FOR_AT_CONFIRM.name(), LitecoinACCTv1TradeBot.State.BOB_WAITING_FOR_AT_CONFIRM.value, + creator.getAddress(), atAddress, timestamp, qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + null, null, + SupportedBlockchain.LITECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + foreignAmount, null, null, null, litecoinReceivingAccountInfo); + + return tradeBotData; + } + + private MintingAccountData createMintingAccountData() { + // These don't need to be valid keys - just 32 byte strings for the purposes of testing + byte[] privateKey = new ECKey().getPrivKeyBytes(); + byte[] publicKey = new ECKey().getPrivKeyBytes(); + + return new MintingAccountData(privateKey, publicKey); + } + + private void exportLegacyTradeBotStatesJson(List allTradeBotData) throws IOException, DataException { + JSONArray allTradeBotDataJson = new JSONArray(); + for (TradeBotData tradeBotData : allTradeBotData) { + JSONObject tradeBotDataJson = tradeBotData.toJson(); + allTradeBotDataJson.put(tradeBotDataJson); + } + + Path backupDirectory = HSQLDBImportExport.getExportDirectory(true); + String fileName = Paths.get(backupDirectory.toString(), "TradeBotStates.json").toString(); + FileWriter writer = new FileWriter(fileName); + writer.write(allTradeBotDataJson.toString()); + writer.close(); + } + + private void deleteExportDirectory() { + // Delete archive directory if exists + Path archivePath = Paths.get(Settings.getInstance().getExportPath()); + try { + FileUtils.deleteDirectory(archivePath.toFile()); + } catch (IOException e) { + + } + } + +} diff --git a/src/test/java/org/qortal/test/MemoryPoWTests.java b/src/test/java/org/qortal/test/MemoryPoWTests.java index 2427afb09..662fab191 100644 --- a/src/test/java/org/qortal/test/MemoryPoWTests.java +++ b/src/test/java/org/qortal/test/MemoryPoWTests.java @@ -1,5 +1,6 @@ package org.qortal.test; +import org.junit.Ignore; import org.junit.Test; import org.qortal.crypto.MemoryPoW; @@ -7,6 +8,7 @@ import java.util.Random; +@Ignore public class MemoryPoWTests { private static final int workBufferLength = 8 * 1024 * 1024; diff --git a/src/test/java/org/qortal/test/PresenceTests.java b/src/test/java/org/qortal/test/PresenceTests.java new file mode 100644 index 000000000..b53b72cb2 --- /dev/null +++ b/src/test/java/org/qortal/test/PresenceTests.java @@ -0,0 +1,133 @@ +package org.qortal.test; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.crosschain.BitcoinACCTv1; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.PresenceTransaction; +import org.qortal.transaction.PresenceTransaction.PresenceType; +import org.qortal.transaction.Transaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.utils.NTP; + +import com.google.common.primitives.Longs; + +import static org.junit.Assert.*; + +public class PresenceTests extends Common { + + private static final byte[] BITCOIN_PKH = new byte[20]; + private static final byte[] HASH_OF_SECRET_B = new byte[32]; + + private PrivateKeyAccount signer; + private Repository repository; + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + + this.repository = RepositoryManager.getRepository(); + this.signer = Common.getTestAccount(this.repository, "bob"); + + // We need to create corresponding test trade offer + byte[] creationBytes = BitcoinACCTv1.buildQortalAT(this.signer.getAddress(), BITCOIN_PKH, HASH_OF_SECRET_B, + 0L, 0L, + 7 * 24 * 60 * 60); + + long txTimestamp = NTP.getTime(); + byte[] lastReference = this.signer.getLastReference(); + + long fee = 0; + String name = "QORT-BTC cross-chain trade"; + String description = "Qortal-Bitcoin cross-chain trade"; + String atType = "ACCT"; + String tags = "QORT-BTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, this.signer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, 1L, Asset.QORT); + + Transaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndImportValid(this.repository, deployAtTransactionData, this.signer); + BlockUtils.mintBlock(this.repository); + } + + @After + public void afterTest() throws DataException { + if (this.repository != null) + this.repository.close(); + + this.repository = null; + } + + @Test + public void validityTests() throws DataException { + long timestamp = System.currentTimeMillis(); + byte[] timestampBytes = Longs.toByteArray(timestamp); + + byte[] timestampSignature = this.signer.sign(timestampBytes); + + assertTrue(isValid(Group.NO_GROUP, this.signer, timestamp, timestampSignature)); + + PrivateKeyAccount nonTrader = Common.getTestAccount(repository, "alice"); + assertFalse(isValid(Group.NO_GROUP, nonTrader, timestamp, timestampSignature)); + } + + @Test + public void newestOnlyTests() throws DataException { + long OLDER_TIMESTAMP = System.currentTimeMillis() - 2000L; + long NEWER_TIMESTAMP = OLDER_TIMESTAMP + 1000L; + + PresenceTransaction older = buildPresenceTransaction(Group.NO_GROUP, this.signer, OLDER_TIMESTAMP, null); + older.computeNonce(); + TransactionUtils.signAndImportValid(repository, older.getTransactionData(), this.signer); + + assertTrue(this.repository.getTransactionRepository().exists(older.getTransactionData().getSignature())); + + PresenceTransaction newer = buildPresenceTransaction(Group.NO_GROUP, this.signer, NEWER_TIMESTAMP, null); + newer.computeNonce(); + TransactionUtils.signAndImportValid(repository, newer.getTransactionData(), this.signer); + + assertTrue(this.repository.getTransactionRepository().exists(newer.getTransactionData().getSignature())); + assertFalse(this.repository.getTransactionRepository().exists(older.getTransactionData().getSignature())); + } + + private boolean isValid(int txGroupId, PrivateKeyAccount signer, long timestamp, byte[] timestampSignature) throws DataException { + Transaction transaction = buildPresenceTransaction(txGroupId, signer, timestamp, timestampSignature); + return transaction.isValidUnconfirmed() == ValidationResult.OK; + } + + private PresenceTransaction buildPresenceTransaction(int txGroupId, PrivateKeyAccount signer, long timestamp, byte[] timestampSignature) throws DataException { + int nonce = 0; + + byte[] reference = signer.getLastReference(); + byte[] creatorPublicKey = signer.getPublicKey(); + long fee = 0L; + + if (timestampSignature == null) + timestampSignature = this.signer.sign(Longs.toByteArray(timestamp)); + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, null); + PresenceTransactionData transactionData = new PresenceTransactionData(baseTransactionData, nonce, PresenceType.TRADE_BOT, timestampSignature); + + return new PresenceTransaction(this.repository, transactionData); + } + +} diff --git a/src/test/java/org/qortal/test/PruneTests.java b/src/test/java/org/qortal/test/PruneTests.java new file mode 100644 index 000000000..0914d7942 --- /dev/null +++ b/src/test/java/org/qortal/test/PruneTests.java @@ -0,0 +1,91 @@ +package org.qortal.test; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.BlockMinter; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.AtUtils; +import org.qortal.test.common.Common; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +public class PruneTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testPruning() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks + for (int i = 2; i <= 10; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Make sure that all blocks have full AT state data and data hash + for (Integer i=2; i <= 10; i++) { + BlockData blockData = repository.getBlockRepository().fromHeight(i); + assertNotNull(blockData.getSignature()); + assertEquals(i, blockData.getHeight()); + List atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i); + assertNotNull(atStatesDataList); + assertFalse(atStatesDataList.isEmpty()); + ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(atStatesDataList.get(0).getATAddress(), i); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + } + + // Prune blocks 2-5 + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 5); + assertEquals(4, numBlocksPruned); + repository.getBlockRepository().setBlockPruneHeight(6); + + // Prune AT states for blocks 2-5 + int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 5); + assertEquals(4, numATStatesPruned); + repository.getATRepository().setAtPruneHeight(6); + + // Make sure that blocks 2-5 are now missing block data and AT states data + for (Integer i=2; i <= 5; i++) { + BlockData blockData = repository.getBlockRepository().fromHeight(i); + assertNull(blockData); + List atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i); + assertTrue(atStatesDataList.isEmpty()); + } + + // ... but blocks 6-10 have block data and full AT states data + for (Integer i=6; i <= 10; i++) { + BlockData blockData = repository.getBlockRepository().fromHeight(i); + assertNotNull(blockData.getSignature()); + List atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i); + assertNotNull(atStatesDataList); + assertFalse(atStatesDataList.isEmpty()); + ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(atStatesDataList.get(0).getATAddress(), i); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + } + } + } + +} diff --git a/src/test/java/org/qortal/test/RepositoryTests.java b/src/test/java/org/qortal/test/RepositoryTests.java index c9532d27a..bb6510d53 100644 --- a/src/test/java/org/qortal/test/RepositoryTests.java +++ b/src/test/java/org/qortal/test/RepositoryTests.java @@ -3,7 +3,12 @@ import org.junit.Before; import org.junit.Test; import org.qortal.account.Account; +import org.qortal.account.PublicKeyAccount; import org.qortal.asset.Asset; +import org.qortal.crosschain.BitcoinACCTv1; +import org.qortal.crypto.Crypto; +import org.qortal.data.account.AccountBalanceData; +import org.qortal.data.account.AccountData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -13,10 +18,15 @@ import static org.junit.Assert.*; +import java.lang.reflect.Field; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -71,7 +81,7 @@ public void testAccessAfterClose() throws DataException { } @Test - public void testDeadlock() throws DataException { + public void testDeadlock() { // Open connection 1 try (final Repository repository1 = RepositoryManager.getRepository()) { @@ -92,18 +102,175 @@ public void testDeadlock() throws DataException { // Update account in 1 account1.setConfirmedBalance(Asset.QORT, 5678L); repository1.saveChanges(); + } catch (DataException e) { + fail("deadlock bug"); + } + } + + @Test + public void testUpdateReadDeadlock() { + // Open connection 1 + try (final Repository repository1 = RepositoryManager.getRepository()) { + // Mint blocks so we have data (online account signatures) to work with + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository1); + + // Perform database 'update', but don't commit at this stage + repository1.getBlockRepository().trimOldOnlineAccountsSignatures(1, 10); + + // Open connection 2 + try (final Repository repository2 = RepositoryManager.getRepository()) { + // Perform database read on same blocks - this should not deadlock + repository2.getBlockRepository().getTimestampFromHeight(5); + } + + // Save updates - this should not deadlock + repository1.saveChanges(); + } catch (DataException e) { + fail("deadlock bug"); + } + } + + @Test + public void testTrimDeadlock() { + ExecutorService executor = Executors.newCachedThreadPool(); + CountDownLatch readyLatch = new CountDownLatch(1); + CountDownLatch updateLatch = new CountDownLatch(1); + CountDownLatch syncLatch = new CountDownLatch(1); + + // Open connection 1 + try (final HSQLDBRepository repository1 = (HSQLDBRepository) RepositoryManager.getRepository()) { + // Read AT states trim height + int atTrimHeight = repository1.getATRepository().getAtTrimHeight(); + repository1.discardChanges(); + + // Open connection 2 + try (final HSQLDBRepository repository2 = (HSQLDBRepository) RepositoryManager.getRepository()) { + // Read online signatures trim height + int onlineSignaturesTrimHeight = repository2.getBlockRepository().getOnlineAccountsSignaturesTrimHeight(); + repository2.discardChanges(); + + Future f2 = executor.submit(() -> { + Object trimHeightsLock = extractTrimHeightsLock(repository2); + System.out.println(String.format("f2: repository2's trimHeightsLock object: %s", trimHeightsLock)); + + // Update online signatures trim height (implicit commit) + synchronized (trimHeightsLock) { + try { + System.out.println("f2: updating online signatures trim height..."); + // simulate: repository2.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(onlineSignaturesTrimHeight); + String updateSql = "UPDATE DatabaseInfo SET online_signatures_trim_height = ?"; + PreparedStatement pstmt = repository2.prepareStatement(updateSql); + pstmt.setInt(1, onlineSignaturesTrimHeight); + pstmt.executeUpdate(); + // But no commit/saveChanges yet to force HSQLDB error + + System.out.println("f2: readyLatch.countDown()"); + readyLatch.countDown(); + + // wait for other thread to be ready to hit sync block + System.out.println("f2: waiting for f1 syncLatch..."); + syncLatch.await(); + + // hang on to trimHeightsLock to force other thread to wait (if code is correct), or to fail (if code is faulty) + System.out.println("f2: updateLatch.await()"); + if (!updateLatch.await(500L, TimeUnit.MILLISECONDS)) { // long enough for other thread to reach synchronized block + // wait period expired suggesting no concurrent access, i.e. code is correct + System.out.println("f2: updateLatch.await() timed out"); + + System.out.println("f2: saveChanges()"); + repository2.saveChanges(); + + return Boolean.TRUE; + } + + System.out.println("f2: saveChanges()"); + repository2.saveChanges(); + + // Early exit from wait period suggests concurrent access, i.e. code faulty + return Boolean.FALSE; + } catch (InterruptedException | SQLException e) { + System.out.println("f2: exception: " + e.getMessage()); + return Boolean.FALSE; + } + } + }); + + System.out.println("waiting for f2 readyLatch..."); + readyLatch.await(); + System.out.println("launching f1..."); + + Future f1 = executor.submit(() -> { + Object trimHeightsLock = extractTrimHeightsLock(repository1); + System.out.println(String.format("f1: repository1's trimHeightsLock object: %s", trimHeightsLock)); + + System.out.println("f1: syncLatch.countDown()"); + syncLatch.countDown(); + + // Update AT states trim height (implicit commit) + synchronized (trimHeightsLock) { + try { + System.out.println("f1: updating AT trim height..."); + // simulate: repository1.getATRepository().setAtTrimHeight(atTrimHeight); + String updateSql = "UPDATE DatabaseInfo SET AT_trim_height = ?"; + PreparedStatement pstmt = repository1.prepareStatement(updateSql); + pstmt.setInt(1, atTrimHeight); + pstmt.executeUpdate(); + System.out.println("f1: saveChanges()"); + repository1.saveChanges(); + + System.out.println("f1: updateLatch.countDown()"); + updateLatch.countDown(); + + return Boolean.TRUE; + } catch (SQLException e) { + System.out.println("f1: exception: " + e.getMessage()); + return Boolean.FALSE; + } + } + }); + + if (Boolean.TRUE != f1.get()) + fail("concurrency bug - simultaneous update of DatabaseInfo table"); + + if (Boolean.TRUE != f2.get()) + fail("concurrency bug - not synchronized on same object?"); + } catch (InterruptedException e) { + fail("concurrency bug: " + e.getMessage()); + } catch (ExecutionException e) { + fail("concurrency bug: " + e.getMessage()); + } + } catch (DataException e) { + fail("database bug"); + } + } + + private static Object extractTrimHeightsLock(HSQLDBRepository repository) { + try { + Field trimHeightsLockField = repository.getClass().getDeclaredField("trimHeightsLock"); + trimHeightsLockField.setAccessible(true); + return trimHeightsLockField.get(repository); + } catch (IllegalArgumentException | NoSuchFieldException | SecurityException | IllegalAccessException e) { + fail(); + return null; } } /** Check that the sub-query used to fetch highest block height is optimized by HSQLDB. */ @Test public void testBlockHeightSpeed() throws DataException, SQLException { + final int mintBlockCount = 10000; + try (final Repository repository = RepositoryManager.getRepository()) { // Mint some blocks - System.out.println("Minting test blocks - should take approx. 30 seconds..."); - for (int i = 0; i < 30000; ++i) + System.out.println(String.format("Minting %d test blocks - should take approx. 10 seconds...", mintBlockCount)); + + long beforeBigMint = System.currentTimeMillis(); + for (int i = 0; i < mintBlockCount; ++i) BlockUtils.mintBlock(repository); + System.out.println(String.format("Minting %d blocks actually took %d seconds", mintBlockCount, (System.currentTimeMillis() - beforeBigMint) / 1000L)); + final HSQLDBRepository hsqldb = (HSQLDBRepository) repository; // Too slow: @@ -158,6 +325,232 @@ public void testInterrupt() { } } + /** + * Test HSQLDB bug-fix for INSERT INTO...ON DUPLICATE KEY UPDATE... bug + *

+ * @see Behaviour of 'ON DUPLICATE KEY UPDATE' SourceForge discussion + */ + @Test + public void testOnDuplicateKeyUpdateBugFix() throws SQLException, DataException { + ResultSet resultSet; + + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + hsqldb.prepareStatement("DROP TABLE IF EXISTS bugtest").execute(); + hsqldb.prepareStatement("CREATE TABLE bugtest (id INT NOT NULL, counter INT NOT NULL, PRIMARY KEY(id))").execute(); + + // No existing row, so new row's "counter" is set to value from VALUES clause, i.e. 1 + hsqldb.prepareStatement("INSERT INTO bugtest (id, counter) VALUES (1, 1) ON DUPLICATE KEY UPDATE counter = counter + 1").execute(); + resultSet = hsqldb.checkedExecute("SELECT counter FROM bugtest WHERE id = 1"); + assertNotNull(resultSet); + assertEquals(1, resultSet.getInt(1)); + + // Prior to bug-fix, "counter = counter + 1" would always use the 100 from VALUES, instead of existing row's value, for "counter" + hsqldb.prepareStatement("INSERT INTO bugtest (id, counter) VALUES (1, 100) ON DUPLICATE KEY UPDATE counter = counter + 1").execute(); + resultSet = hsqldb.checkedExecute("SELECT counter FROM bugtest WHERE id = 1"); + assertNotNull(resultSet); + // Prior to bug-fix, this would be 100 + 1 = 101 + assertEquals(2, resultSet.getInt(1)); + } + } + + /** + * Test HSQLDB bug-fix for "General Error" in non-fully-qualified columns inside LATERAL() + *

+ * @see #1580 General error with LATERAL and transitive join column SourceForge ticket + */ + @Test + public void testOnLateralGeneralError() { + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + hsqldb.prepareStatement("DROP TABLE IF EXISTS tableA").execute(); + hsqldb.prepareStatement("DROP TABLE IF EXISTS tableB").execute(); + hsqldb.prepareStatement("DROP TABLE IF EXISTS tableC").execute(); + + hsqldb.prepareStatement("CREATE TABLE tableA (col1 INT)").execute(); + hsqldb.prepareStatement("CREATE TABLE tableB (col1 INT)").execute(); + hsqldb.prepareStatement("CREATE TABLE tableC (col2 INT, PRIMARY KEY (col2))").execute(); + + // Prior to bug-fix #1580 this would throw a General Error SQL Exception + hsqldb.prepareStatement("SELECT col3 FROM tableA JOIN tableB USING (col1) CROSS JOIN LATERAL(SELECT col2 FROM tableC WHERE col2 = col1) AS tableC (col3)").execute(); + } catch (SQLException | DataException e) { + fail("HSQLDB bug #1580"); + } + } + + /** Specifically test LATERAL() usage in Asset repository */ + @Test + public void testAssetLateral() { + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + List assetIds = Collections.emptyList(); + List otherAssetIds = Collections.emptyList(); + Integer limit = null; + Integer offset = null; + Boolean reverse = null; + + hsqldb.getAssetRepository().getRecentTrades(assetIds, otherAssetIds, limit, offset, reverse); + } catch (DataException e) { + fail("HSQLDB bug #1580"); + } + } + + /** Specifically test LATERAL() usage in AT repository */ + @Test + public void testAtLateral() { + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + byte[] codeHash = BitcoinACCTv1.CODE_BYTES_HASH; + Boolean isFinished = null; + Integer dataByteOffset = null; + Long expectedValue = null; + Integer minimumFinalHeight = 2; + Integer limit = null; + Integer offset = null; + Boolean reverse = null; + + hsqldb.getATRepository().getMatchingFinalATStates(codeHash, isFinished, dataByteOffset, expectedValue, minimumFinalHeight, limit, offset, reverse); + } catch (DataException e) { + fail("HSQLDB bug #1580"); + } + } + + /** Specifically test LATERAL() usage in Chat repository */ + @Test + public void testChatLateral() { + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + String address = Crypto.toAddress(new byte[32]); + + hsqldb.getChatRepository().getActiveChats(address); + } catch (DataException e) { + fail("HSQLDB bug #1580"); + } + } + + /** Test batched DELETE */ + @Test + public void testBatchedDelete() { + // Generate test data + List batchedObjects = new ArrayList<>(); + for (int i = 0; i < 100; ++i) + batchedObjects.add(new Object[] { String.valueOf(i), 1L }); + + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + hsqldb.deleteBatch("AccountBalances", "account = ? AND asset_id = ?", batchedObjects); + } catch (DataException | SQLException e) { + fail("Batched delete failed: " + e.getMessage()); + } + } + + @Test + public void testDefrag() throws DataException, TimeoutException { + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + + this.populateWithRandomData(hsqldb); + + hsqldb.performPeriodicMaintenance(10 * 1000L); + + } + } + + @Test + public void testDefragOnDisk() throws DataException, TimeoutException { + Common.useSettingsAndDb(testSettingsFilename, false); + + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + + this.populateWithRandomData(hsqldb); + + hsqldb.performPeriodicMaintenance(10 * 1000L); + + } + } + + @Test + public void testMultipleDefrags() throws DataException, TimeoutException { + // Mint some more blocks to populate the database + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + + this.populateWithRandomData(hsqldb); + + for (int i = 0; i < 10; i++) { + hsqldb.performPeriodicMaintenance(10 * 1000L); + } + } + } + + @Test + public void testMultipleDefragsOnDisk() throws DataException, TimeoutException { + Common.useSettingsAndDb(testSettingsFilename, false); + + // Mint some more blocks to populate the database + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + + this.populateWithRandomData(hsqldb); + + for (int i = 0; i < 10; i++) { + hsqldb.performPeriodicMaintenance(10 * 1000L); + } + } + } + + @Test + public void testMultipleDefragsWithDifferentData() throws DataException, TimeoutException { + for (int i=0; i<10; i++) { + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + + this.populateWithRandomData(hsqldb); + hsqldb.performPeriodicMaintenance(10 * 1000L); + } + } + } + + @Test + public void testMultipleDefragsOnDiskWithDifferentData() throws DataException, TimeoutException { + Common.useSettingsAndDb(testSettingsFilename, false); + + for (int i=0; i<10; i++) { + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + + this.populateWithRandomData(hsqldb); + hsqldb.performPeriodicMaintenance(10 * 1000L); + } + } + } + + private void populateWithRandomData(HSQLDBRepository repository) throws DataException { + Random random = new Random(); + + System.out.println("Creating random accounts..."); + + // Generate some random accounts + List accounts = new ArrayList<>(); + for (int ai = 0; ai < 20; ++ai) { + byte[] publicKey = new byte[32]; + random.nextBytes(publicKey); + + PublicKeyAccount account = new PublicKeyAccount(repository, publicKey); + accounts.add(account); + + AccountData accountData = new AccountData(account.getAddress()); + repository.getAccountRepository().ensureAccount(accountData); + } + repository.saveChanges(); + + System.out.println("Creating random balances..."); + + // Fill with lots of random balances + for (int i = 0; i < 100000; ++i) { + Account account = accounts.get(random.nextInt(accounts.size())); + int assetId = random.nextInt(2); + long balance = random.nextInt(100000); + + AccountBalanceData accountBalanceData = new AccountBalanceData(account.getAddress(), assetId, balance); + repository.getAccountRepository().save(accountBalanceData); + + // Maybe mint a block to change height + if (i > 0 && (i % 1000) == 0) + BlockUtils.mintBlock(repository); + } + repository.saveChanges(); + } + public static void hsqldbSleep(int millis) throws SQLException { System.out.println(String.format("HSQLDB sleep() thread ID: %s", Thread.currentThread().getId())); diff --git a/src/test/java/org/qortal/test/SerializationTests.java b/src/test/java/org/qortal/test/SerializationTests.java index 0632495f1..5a9fffa83 100644 --- a/src/test/java/org/qortal/test/SerializationTests.java +++ b/src/test/java/org/qortal/test/SerializationTests.java @@ -1,5 +1,6 @@ package org.qortal.test; +import org.junit.Ignore; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; import org.qortal.data.transaction.TransactionData; @@ -172,4 +173,4 @@ public void testNegativeBigDecimal() throws IOException { assertEqualBigDecimals("Deserialized BigDecimal has incorrect value", amount, newAmount); } -} \ No newline at end of file +} diff --git a/src/test/java/org/qortal/test/TransactionSearchTests.java b/src/test/java/org/qortal/test/TransactionSearchTests.java new file mode 100644 index 000000000..8cc756464 --- /dev/null +++ b/src/test/java/org/qortal/test/TransactionSearchTests.java @@ -0,0 +1,76 @@ +package org.qortal.test; + +import static org.junit.Assert.*; + +import java.util.List; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.AccountUtils; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.transaction.Transaction.TransactionType; + +public class TransactionSearchTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testFindingSpecificTransactionsWithinHeight() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + PrivateKeyAccount chloe = Common.getTestAccount(repository, "chloe"); + + // Block 2 + BlockUtils.mintBlock(repository); + + // Block 3 + AccountUtils.pay(repository, alice, chloe.getAddress(), 1234L); + + // Block 4 + AccountUtils.pay(repository, chloe, alice.getAddress(), 5678L); + + // Block 5 + BlockUtils.mintBlock(repository); + + List signatures; + + // No transactions with this type + signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(TransactionType.GROUP_KICK, null, null, null); + assertEquals(0, signatures.size()); + + // 2 payments + signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(TransactionType.PAYMENT, null, null, null); + assertEquals(2, signatures.size()); + + // 1 payment by Alice + signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(TransactionType.PAYMENT, alice.getPublicKey(), null, null); + assertEquals(1, signatures.size()); + + // 1 transaction by Chloe + signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, chloe.getPublicKey(), null, null); + assertEquals(1, signatures.size()); + + // 1 transaction from blocks 4 onwards + signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, 4, null); + assertEquals(1, signatures.size()); + + // 1 transaction from blocks 2 to 3 + signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, 2, 3); + assertEquals(1, signatures.size()); + + // No transaction of this type from blocks 2 to 5 + signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(TransactionType.ISSUE_ASSET, null, 2, 5); + assertEquals(0, signatures.size()); + } + + } + +} diff --git a/src/test/java/org/qortal/test/TransferPrivsTests.java b/src/test/java/org/qortal/test/TransferPrivsTests.java index f95e0599d..3ed3ad161 100644 --- a/src/test/java/org/qortal/test/TransferPrivsTests.java +++ b/src/test/java/org/qortal/test/TransferPrivsTests.java @@ -2,12 +2,13 @@ import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.qortal.account.Account; import org.qortal.account.PrivateKeyAccount; import org.qortal.account.PublicKeyAccount; import org.qortal.block.BlockChain; -import org.qortal.block.BlockMinter; +import org.qortal.controller.BlockMinter; import org.qortal.data.account.AccountData; import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.PaymentTransactionData; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Random; +@Ignore(value = "Doesn't work, to be fixed later") public class TransferPrivsTests extends Common { private static List cumulativeBlocksByLevel; diff --git a/src/test/java/org/qortal/test/api/AddressesApiTests.java b/src/test/java/org/qortal/test/api/AddressesApiTests.java index c1d28cb68..1510f63ff 100644 --- a/src/test/java/org/qortal/test/api/AddressesApiTests.java +++ b/src/test/java/org/qortal/test/api/AddressesApiTests.java @@ -5,6 +5,7 @@ import java.util.Collections; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.qortal.api.resource.AddressesResource; import org.qortal.test.common.ApiCommon; @@ -24,6 +25,7 @@ public void testGetAccountInfo() { } @Test + @Ignore(value = "Doesn't work, to be fixed later") public void testGetOnlineAccounts() { assertNotNull(this.addressesResource.getOnlineAccounts()); } diff --git a/src/test/java/org/qortal/test/api/AdminApiTests.java b/src/test/java/org/qortal/test/api/AdminApiTests.java index 4aa2ca3b6..b3e6da036 100644 --- a/src/test/java/org/qortal/test/api/AdminApiTests.java +++ b/src/test/java/org/qortal/test/api/AdminApiTests.java @@ -2,15 +2,24 @@ import static org.junit.Assert.*; +import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.Before; import org.junit.Test; import org.qortal.api.resource.AdminResource; +import org.qortal.repository.DataException; +import org.qortal.settings.Settings; import org.qortal.test.common.ApiCommon; +import org.qortal.test.common.Common; public class AdminApiTests extends ApiCommon { private AdminResource adminResource; + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + @Before public void buildResource() { this.adminResource = (AdminResource) ApiCommon.buildResource(AdminResource.class); @@ -22,8 +31,11 @@ public void testInfo() { } @Test - public void testSummary() { - assertNotNull(this.adminResource.summary()); + public void testSummary() throws IllegalAccessException { + // Set localAuthBypassEnabled to true, since we don't need to test authentication here + FieldUtils.writeField(Settings.getInstance(), "localAuthBypassEnabled", true, true); + + assertNotNull(this.adminResource.summary("testApiKey")); } @Test diff --git a/src/test/java/org/qortal/test/api/ArbitraryApiTests.java b/src/test/java/org/qortal/test/api/ArbitraryApiTests.java index 2b80fb513..e4f27db61 100644 --- a/src/test/java/org/qortal/test/api/ArbitraryApiTests.java +++ b/src/test/java/org/qortal/test/api/ArbitraryApiTests.java @@ -6,6 +6,7 @@ import org.junit.Test; import org.qortal.api.resource.ArbitraryResource; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; +import org.qortal.arbitrary.misc.Service; import org.qortal.test.common.ApiCommon; public class ArbitraryApiTests extends ApiCommon { @@ -22,22 +23,24 @@ public void testSearch() { Integer[] startingBlocks = new Integer[] { null, 0, 1, 999999999 }; Integer[] blockLimits = new Integer[] { null, 0, 1, 999999999 }; Integer[] txGroupIds = new Integer[] { null, 0, 1, 999999999 }; - Integer[] services = new Integer[] { null, 0, 1, 999999999 }; + Service[] services = new Service[] { Service.WEBSITE, Service.GIT_REPOSITORY, Service.BLOG_COMMENT }; + String[] names = new String[] { null, "Test" }; String[] addresses = new String[] { null, this.aliceAddress }; ConfirmationStatus[] confirmationStatuses = new ConfirmationStatus[] { ConfirmationStatus.UNCONFIRMED, ConfirmationStatus.CONFIRMED, ConfirmationStatus.BOTH }; for (Integer startBlock : startingBlocks) for (Integer blockLimit : blockLimits) for (Integer txGroupId : txGroupIds) - for (Integer service : services) - for (String address : addresses) - for (ConfirmationStatus confirmationStatus : confirmationStatuses) { - if (confirmationStatus != ConfirmationStatus.CONFIRMED && (startBlock != null || blockLimit != null)) - continue; - - assertNotNull(this.arbitraryResource.searchTransactions(startBlock, blockLimit, txGroupId, service, address, confirmationStatus, 20, null, null)); - assertNotNull(this.arbitraryResource.searchTransactions(startBlock, blockLimit, txGroupId, service, address, confirmationStatus, 1, 1, true)); - } + for (Service service : services) + for (String name : names) + for (String address : addresses) + for (ConfirmationStatus confirmationStatus : confirmationStatuses) { + if (confirmationStatus != ConfirmationStatus.CONFIRMED && (startBlock != null || blockLimit != null)) + continue; + + assertNotNull(this.arbitraryResource.searchTransactions(startBlock, blockLimit, txGroupId, service, name, address, confirmationStatus, 20, null, null)); + assertNotNull(this.arbitraryResource.searchTransactions(startBlock, blockLimit, txGroupId, service, name, address, confirmationStatus, 1, 1, true)); + } } } diff --git a/src/test/java/org/qortal/test/api/BlockApiTests.java b/src/test/java/org/qortal/test/api/BlockApiTests.java index 384c9858b..47d5318a2 100644 --- a/src/test/java/org/qortal/test/api/BlockApiTests.java +++ b/src/test/java/org/qortal/test/api/BlockApiTests.java @@ -7,8 +7,10 @@ import java.util.List; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.api.ApiError; import org.qortal.api.resource.BlocksResource; import org.qortal.block.GenesisBlock; import org.qortal.repository.DataException; @@ -39,7 +41,7 @@ public void testGetBlock() throws DataException { byte[] signatureBytes = GenesisBlock.getInstance(repository).getSignature(); String signature = Base58.encode(signatureBytes); - assertNotNull(this.blocksResource.getBlock(signature)); + assertNotNull(this.blocksResource.getBlock(signature, true)); } } @@ -71,17 +73,31 @@ public void testGetBlockHeight() throws DataException { @Test public void testGetBlockByHeight() { - assertNotNull(this.blocksResource.getByHeight(1)); + assertNotNull(this.blocksResource.getByHeight(1, true)); } @Test - public void testGetBlockByTimestamp() { - assertNotNull(this.blocksResource.getByTimestamp(System.currentTimeMillis())); + @Ignore(value = "Doesn't work, to be fixed later") + public void testGetBlockByTimestamp() throws DataException { + assertNotNull(this.blocksResource.getByTimestamp(System.currentTimeMillis(), false)); } @Test public void testGetBlockRange() { assertNotNull(this.blocksResource.getBlockRange(1, 1)); + + List testValues = Arrays.asList(null, Integer.valueOf(1)); + + for (Integer startHeight : testValues) + for (Integer endHeight : testValues) + for (Integer count : testValues) { + if (startHeight != null && endHeight != null && count != null) { + assertApiError(ApiError.INVALID_CRITERIA, () -> this.blocksResource.getBlockSummaries(startHeight, endHeight, count)); + continue; + } + + assertNotNull(this.blocksResource.getBlockSummaries(startHeight, endHeight, count)); + } } @Test diff --git a/src/test/java/org/qortal/test/api/CrossChainApiTests.java b/src/test/java/org/qortal/test/api/CrossChainApiTests.java new file mode 100644 index 000000000..d4f25bce6 --- /dev/null +++ b/src/test/java/org/qortal/test/api/CrossChainApiTests.java @@ -0,0 +1,42 @@ +package org.qortal.test.api; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.api.ApiError; +import org.qortal.api.resource.CrossChainResource; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.test.common.ApiCommon; + +public class CrossChainApiTests extends ApiCommon { + + private static final SupportedBlockchain SPECIFIC_BLOCKCHAIN = null; + + private CrossChainResource crossChainResource; + + @Before + public void buildResource() { + this.crossChainResource = (CrossChainResource) ApiCommon.buildResource(CrossChainResource.class); + } + + @Test + public void testGetTradeOffers() { + assertNoApiError((limit, offset, reverse) -> this.crossChainResource.getTradeOffers(SPECIFIC_BLOCKCHAIN, limit, offset, reverse)); + } + + @Test + public void testGetCompletedTrades() { + long minimumTimestamp = System.currentTimeMillis(); + assertNoApiError((limit, offset, reverse) -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, minimumTimestamp, limit, offset, reverse)); + } + + @Test + public void testInvalidGetCompletedTrades() { + Integer limit = null; + Integer offset = null; + Boolean reverse = null; + + assertApiError(ApiError.INVALID_CRITERIA, () -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, -1L /*minimumTimestamp*/, limit, offset, reverse)); + assertApiError(ApiError.INVALID_CRITERIA, () -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, 0L /*minimumTimestamp*/, limit, offset, reverse)); + } + +} diff --git a/src/test/java/org/qortal/test/api/NamesApiTests.java b/src/test/java/org/qortal/test/api/NamesApiTests.java index 2d31c8c27..962f1b925 100644 --- a/src/test/java/org/qortal/test/api/NamesApiTests.java +++ b/src/test/java/org/qortal/test/api/NamesApiTests.java @@ -16,6 +16,8 @@ import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.utils.NTP; public class NamesApiTests extends ApiCommon { @@ -47,6 +49,7 @@ public void testGetNamesByAddress() throws DataException { String name = "test-name"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); assertNotNull(this.namesResource.getNamesByAddress(alice.getAddress(), null, null, null)); @@ -62,6 +65,7 @@ public void testGetName() throws DataException { String name = "test-name"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); assertNotNull(this.namesResource.getName(name)); @@ -77,6 +81,7 @@ public void testGetAllAssets() throws DataException { long price = 1_23456789L; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Sell-name diff --git a/src/test/java/org/qortal/test/apps/BTCACCTTests.java b/src/test/java/org/qortal/test/apps/BTCACCTTests.java deleted file mode 100644 index 499cf7433..000000000 --- a/src/test/java/org/qortal/test/apps/BTCACCTTests.java +++ /dev/null @@ -1,327 +0,0 @@ -package org.qortal.test.apps; - -import java.io.File; -import java.net.UnknownHostException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; -import java.security.Security; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ExecutionException; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.ECKey; -import org.bitcoinj.core.InsufficientMoneyException; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.core.Sha256Hash; -import org.bitcoinj.core.Transaction; -import org.bitcoinj.core.Transaction.SigHash; -import org.bitcoinj.core.TransactionBroadcast; -import org.bitcoinj.core.TransactionInput; -import org.bitcoinj.core.TransactionOutPoint; -import org.bitcoinj.crypto.TransactionSignature; -import org.bitcoinj.kits.WalletAppKit; -import org.bitcoinj.params.TestNet3Params; -import org.bitcoinj.script.Script; -import org.bitcoinj.script.Script.ScriptType; -import org.bitcoinj.script.ScriptBuilder; -import org.bitcoinj.script.ScriptChunk; -import org.bitcoinj.script.ScriptOpCodes; -import org.bitcoinj.wallet.WalletTransaction.Pool; -import org.bouncycastle.jce.provider.BouncyCastleProvider; - -import com.google.common.hash.HashCode; -import com.google.common.primitives.Bytes; - -/** - * Initiator must be Qortal-chain so that initiator can send initial message to BTC P2SH then Qortal can scan for P2SH add send corresponding message to Qortal AT. - * - * Initiator (wants QORT, has BTC) - * Funds BTC P2SH address - * - * Responder (has QORT, wants BTC) - * Builds Qortal ACCT AT and funds it with QORT - * - * Initiator sends recipient+secret+script as input to BTC P2SH address, releasing BTC amount - fees to responder - * - * Qortal nodes scan for P2SH output, checks amount and recipient and if ok sends secret to Qortal ACCT AT - * (Or it's possible to feed BTC transaction details into Qortal AT so it can check them itself?) - * - * Qortal ACCT AT sends its QORT to initiator - * - */ - -public class BTCACCTTests { - - private static final long TIMEOUT = 600L; - private static final Coin sendValue = Coin.valueOf(6_000L); - private static final Coin fee = Coin.valueOf(2_000L); - - private static final byte[] senderPrivKeyBytes = HashCode.fromString("027fb5828c5e201eaf6de4cd3b0b340d16a191ef848cd691f35ef8f727358c9c").asBytes(); - private static final byte[] recipientPrivKeyBytes = HashCode.fromString("ec199a4abc9d3bf024349e397535dfee9d287e174aeabae94237eb03a0118c03").asBytes(); - - // The following need to be updated manually - private static final String prevTxHash = "70ee97f20afea916c2e7b47f6abf3c75f97c4c2251b4625419406a2dd47d16b5"; - private static final Coin prevTxBalance = Coin.valueOf(562_000L); // This is NOT the amount but the unspent balance - private static final long prevTxOutputIndex = 1L; - - // For when we want to re-run - private static final byte[] prevSecret = HashCode.fromString("30a13291e350214bea5318f990b77bc11d2cb709f7c39859f248bef396961dcc").asBytes(); - private static final long prevLockTime = 1539347892L; - private static final boolean usePreviousFundingTx = false; - - private static final boolean doRefundNotRedeem = false; - - public static void main(String[] args) throws NoSuchAlgorithmException, InsufficientMoneyException, InterruptedException, ExecutionException, UnknownHostException { - Security.insertProviderAt(new BouncyCastleProvider(), 0); - - byte[] secret = new byte[32]; - new SecureRandom().nextBytes(secret); - - if (usePreviousFundingTx) - secret = prevSecret; - - System.out.println("Secret: " + HashCode.fromBytes(secret).toString()); - - MessageDigest sha256Digester = MessageDigest.getInstance("SHA-256"); - - byte[] secretHash = sha256Digester.digest(secret); - String secretHashHex = HashCode.fromBytes(secretHash).toString(); - - System.out.println("SHA256(secret): " + secretHashHex); - - NetworkParameters params = TestNet3Params.get(); - // NetworkParameters params = RegTestParams.get(); - System.out.println("Network: " + params.getId()); - - WalletAppKit kit = new WalletAppKit(params, new File("."), "btc-tests"); - - kit.setBlockingStartup(false); - kit.startAsync(); - kit.awaitRunning(); - - long now = System.currentTimeMillis() / 1000L; - long lockTime = now + TIMEOUT; - - if (usePreviousFundingTx) - lockTime = prevLockTime; - - System.out.println("LockTime: " + lockTime); - - ECKey senderKey = ECKey.fromPrivate(senderPrivKeyBytes); - kit.wallet().importKey(senderKey); - ECKey recipientKey = ECKey.fromPrivate(recipientPrivKeyBytes); - kit.wallet().importKey(recipientKey); - - byte[] senderPubKey = senderKey.getPubKey(); - System.out.println("Sender address: " + Address.fromKey(params, senderKey, ScriptType.P2PKH).toString()); - System.out.println("Sender pubkey: " + HashCode.fromBytes(senderPubKey).toString()); - - byte[] recipientPubKey = recipientKey.getPubKey(); - System.out.println("Recipient address: " + Address.fromKey(params, recipientKey, ScriptType.P2PKH).toString()); - System.out.println("Recipient pubkey: " + HashCode.fromBytes(recipientPubKey).toString()); - - byte[] redeemScriptBytes = buildRedeemScript(secret, senderPubKey, recipientPubKey, lockTime); - System.out.println("Redeem script: " + HashCode.fromBytes(redeemScriptBytes).toString()); - - byte[] redeemScriptHash = hash160(redeemScriptBytes); - - Address p2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - System.out.println("P2SH address: " + p2shAddress.toString()); - - // Send amount to P2SH address - Transaction fundingTransaction = buildFundingTransaction(params, Sha256Hash.wrap(prevTxHash), prevTxOutputIndex, prevTxBalance, senderKey, - sendValue.add(fee), redeemScriptHash); - - System.out.println("Sending " + sendValue.add(fee).toPlainString() + " to " + p2shAddress.toString()); - if (!usePreviousFundingTx) - broadcastWithConfirmation(kit, fundingTransaction); - - if (doRefundNotRedeem) { - // Refund - System.out.println("Refunding " + sendValue.toPlainString() + " back to " + Address.fromKey(params, senderKey, ScriptType.P2PKH).toString()); - - now = System.currentTimeMillis() / 1000L; - long refundLockTime = now - 60 * 30; // 30 minutes in the past, needs to before 'now' and before "median block time" (median of previous 11 block - // timestamps) - if (refundLockTime < lockTime) - throw new RuntimeException("Too soon to refund"); - - TransactionOutPoint fundingOutPoint = new TransactionOutPoint(params, 0, fundingTransaction); - Transaction refundTransaction = buildRefundTransaction(params, fundingOutPoint, senderKey, sendValue, redeemScriptBytes, refundLockTime); - broadcastWithConfirmation(kit, refundTransaction); - } else { - // Redeem - System.out.println("Redeeming " + sendValue.toPlainString() + " to " + Address.fromKey(params, recipientKey, ScriptType.P2PKH).toString()); - - TransactionOutPoint fundingOutPoint = new TransactionOutPoint(params, 0, fundingTransaction); - Transaction redeemTransaction = buildRedeemTransaction(params, fundingOutPoint, recipientKey, sendValue, secret, redeemScriptBytes); - broadcastWithConfirmation(kit, redeemTransaction); - } - - kit.wallet().cleanup(); - - for (Transaction transaction : kit.wallet().getTransactionPool(Pool.PENDING).values()) - System.out.println("Pending tx: " + transaction.getTxId().toString()); - } - - private static final byte[] redeemScript1 = HashCode.fromString("76a820").asBytes(); - private static final byte[] redeemScript2 = HashCode.fromString("87637576a914").asBytes(); - private static final byte[] redeemScript3 = HashCode.fromString("88ac6704").asBytes(); - private static final byte[] redeemScript4 = HashCode.fromString("b17576a914").asBytes(); - private static final byte[] redeemScript5 = HashCode.fromString("88ac68").asBytes(); - - private static byte[] buildRedeemScript(byte[] secret, byte[] senderPubKey, byte[] recipientPubKey, long lockTime) { - try { - MessageDigest sha256Digester = MessageDigest.getInstance("SHA-256"); - - byte[] secretHash = sha256Digester.digest(secret); - byte[] senderPubKeyHash = hash160(senderPubKey); - byte[] recipientPubKeyHash = hash160(recipientPubKey); - - return Bytes.concat(redeemScript1, secretHash, redeemScript2, recipientPubKeyHash, redeemScript3, toLEByteArray((int) (lockTime & 0xffffffffL)), - redeemScript4, senderPubKeyHash, redeemScript5); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("Message digest unsupported", e); - } - } - - private static byte[] hash160(byte[] input) { - try { - MessageDigest rmd160Digester = MessageDigest.getInstance("RIPEMD160"); - MessageDigest sha256Digester = MessageDigest.getInstance("SHA-256"); - - return rmd160Digester.digest(sha256Digester.digest(input)); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("Message digest unsupported", e); - } - } - - private static Transaction buildFundingTransaction(NetworkParameters params, Sha256Hash prevTxHash, long outputIndex, Coin balance, ECKey sigKey, Coin value, - byte[] redeemScriptHash) { - Transaction fundingTransaction = new Transaction(params); - - // Outputs (needed before input so inputs can be signed) - // Fixed amount to P2SH - fundingTransaction.addOutput(value, ScriptBuilder.createP2SHOutputScript(redeemScriptHash)); - // Change to sender - fundingTransaction.addOutput(balance.minus(value).minus(fee), ScriptBuilder.createOutputScript(Address.fromKey(params, sigKey, ScriptType.P2PKH))); - - // Input - // We create fake "to address" scriptPubKey for prev tx so our spending input is P2PKH type - Script fakeScriptPubKey = ScriptBuilder.createOutputScript(Address.fromKey(params, sigKey, ScriptType.P2PKH)); - TransactionOutPoint prevOut = new TransactionOutPoint(params, outputIndex, prevTxHash); - fundingTransaction.addSignedInput(prevOut, fakeScriptPubKey, sigKey); - - return fundingTransaction; - } - - private static Transaction buildRedeemTransaction(NetworkParameters params, TransactionOutPoint fundingOutPoint, ECKey recipientKey, Coin value, byte[] secret, - byte[] redeemScriptBytes) { - Transaction redeemTransaction = new Transaction(params); - redeemTransaction.setVersion(2); - - // Outputs - redeemTransaction.addOutput(value, ScriptBuilder.createOutputScript(Address.fromKey(params, recipientKey, ScriptType.P2PKH))); - - // Input - byte[] recipientPubKey = recipientKey.getPubKey(); - ScriptBuilder scriptBuilder = new ScriptBuilder(); - scriptBuilder.addChunk(new ScriptChunk(recipientPubKey.length, recipientPubKey)); - scriptBuilder.addChunk(new ScriptChunk(secret.length, secret)); - scriptBuilder.addChunk(new ScriptChunk(ScriptOpCodes.OP_PUSHDATA1, redeemScriptBytes)); - byte[] scriptPubKey = scriptBuilder.build().getProgram(); - - TransactionInput input = new TransactionInput(params, null, scriptPubKey, fundingOutPoint); - input.setSequenceNumber(0xffffffffL); // Final - redeemTransaction.addInput(input); - - // Generate transaction signature for input - boolean anyoneCanPay = false; - Sha256Hash hash = redeemTransaction.hashForSignature(0, redeemScriptBytes, SigHash.ALL, anyoneCanPay); - System.out.println("redeem transaction's input hash: " + hash.toString()); - - ECKey.ECDSASignature ecSig = recipientKey.sign(hash); - TransactionSignature txSig = new TransactionSignature(ecSig, SigHash.ALL, anyoneCanPay); - byte[] txSigBytes = txSig.encodeToBitcoin(); - System.out.println("redeem transaction's signature: " + HashCode.fromBytes(txSigBytes).toString()); - - // Prepend signature to input - scriptBuilder.addChunk(0, new ScriptChunk(txSigBytes.length, txSigBytes)); - input.setScriptSig(scriptBuilder.build()); - - return redeemTransaction; - } - - private static Transaction buildRefundTransaction(NetworkParameters params, TransactionOutPoint fundingOutPoint, ECKey senderKey, Coin value, - byte[] redeemScriptBytes, long lockTime) { - Transaction refundTransaction = new Transaction(params); - refundTransaction.setVersion(2); - - // Outputs - refundTransaction.addOutput(value, ScriptBuilder.createOutputScript(Address.fromKey(params, senderKey, ScriptType.P2PKH))); - - // Input - byte[] recipientPubKey = senderKey.getPubKey(); - ScriptBuilder scriptBuilder = new ScriptBuilder(); - scriptBuilder.addChunk(new ScriptChunk(recipientPubKey.length, recipientPubKey)); - scriptBuilder.addChunk(new ScriptChunk(ScriptOpCodes.OP_PUSHDATA1, redeemScriptBytes)); - byte[] scriptPubKey = scriptBuilder.build().getProgram(); - - TransactionInput input = new TransactionInput(params, null, scriptPubKey, fundingOutPoint); - input.setSequenceNumber(0); - refundTransaction.addInput(input); - - // Set locktime after input but before input signature is generated - refundTransaction.setLockTime(lockTime); - - // Generate transaction signature for input - boolean anyoneCanPay = false; - Sha256Hash hash = refundTransaction.hashForSignature(0, redeemScriptBytes, SigHash.ALL, anyoneCanPay); - System.out.println("refund transaction's input hash: " + hash.toString()); - - ECKey.ECDSASignature ecSig = senderKey.sign(hash); - TransactionSignature txSig = new TransactionSignature(ecSig, SigHash.ALL, anyoneCanPay); - byte[] txSigBytes = txSig.encodeToBitcoin(); - System.out.println("refund transaction's signature: " + HashCode.fromBytes(txSigBytes).toString()); - - // Prepend signature to input - scriptBuilder.addChunk(0, new ScriptChunk(txSigBytes.length, txSigBytes)); - input.setScriptSig(scriptBuilder.build()); - - return refundTransaction; - } - - private static void broadcastWithConfirmation(WalletAppKit kit, Transaction transaction) { - System.out.println("Broadcasting tx: " + transaction.getTxId().toString()); - System.out.println("TX hex: " + HashCode.fromBytes(transaction.bitcoinSerialize()).toString()); - - System.out.println("Number of connected peers: " + kit.peerGroup().numConnectedPeers()); - TransactionBroadcast txBroadcast = kit.peerGroup().broadcastTransaction(transaction); - - try { - txBroadcast.future().get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException("Transaction broadcast failed", e); - } - - // wait for confirmation - System.out.println("Waiting for confirmation of tx: " + transaction.getTxId().toString()); - - try { - transaction.getConfidence().getDepthFuture(1).get(); - } catch (CancellationException | ExecutionException | InterruptedException e) { - throw new RuntimeException("Transaction confirmation failed", e); - } - - System.out.println("Confirmed tx: " + transaction.getTxId().toString()); - } - - /** Convert int to little-endian byte array */ - private static byte[] toLEByteArray(int value) { - return new byte[] { (byte) (value), (byte) (value >> 8), (byte) (value >> 16), (byte) (value >> 24) }; - } - -} diff --git a/src/test/java/org/qortal/test/apps/CheckTranslations.java b/src/test/java/org/qortal/test/apps/CheckTranslations.java index faf1727d5..2b59ce84b 100644 --- a/src/test/java/org/qortal/test/apps/CheckTranslations.java +++ b/src/test/java/org/qortal/test/apps/CheckTranslations.java @@ -14,9 +14,9 @@ public class CheckTranslations { private static final String[] SUPPORTED_LANGS = new String[] { "en", "de", "zh", "ru" }; private static final Set SYSTRAY_KEYS = Set.of("AUTO_UPDATE", "APPLYING_UPDATE_AND_RESTARTING", "BLOCK_HEIGHT", - "CHECK_TIME_ACCURACY", "CONNECTING", "CONNECTION", "CONNECTIONS", "CREATING_BACKUP_OF_DB_FILES", "DB_BACKUP", "EXIT", - "MINTING_DISABLED", "MINTING_ENABLED", "NTP_NAG_CAPTION", "NTP_NAG_TEXT_UNIX", "NTP_NAG_TEXT_WINDOWS", - "OPEN_UI", "SYNCHRONIZE_CLOCK", "SYNCHRONIZING_BLOCKCHAIN", "SYNCHRONIZING_CLOCK"); + "BUILD_VERSION", "CHECK_TIME_ACCURACY", "CONNECTING", "CONNECTION", "CONNECTIONS", "CREATING_BACKUP_OF_DB_FILES", + "DB_BACKUP", "DB_CHECKPOINT", "EXIT", "MINTING_DISABLED", "MINTING_ENABLED", "OPEN_UI", "PERFORMING_DB_CHECKPOINT", + "SYNCHRONIZE_CLOCK", "SYNCHRONIZING_BLOCKCHAIN", "SYNCHRONIZING_CLOCK"); private static String failurePrefix; diff --git a/src/test/java/org/qortal/test/apps/DecodeOnlineAccounts.java b/src/test/java/org/qortal/test/apps/DecodeOnlineAccounts.java index 1781f7192..9242c4227 100644 --- a/src/test/java/org/qortal/test/apps/DecodeOnlineAccounts.java +++ b/src/test/java/org/qortal/test/apps/DecodeOnlineAccounts.java @@ -3,7 +3,6 @@ import java.math.BigDecimal; import java.security.Security; -import org.bitcoinj.core.Base58; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; import org.qortal.block.BlockChain; @@ -17,6 +16,7 @@ import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; import org.qortal.transform.block.BlockTransformer; +import org.qortal.utils.Base58; import org.roaringbitmap.IntIterator; import io.druid.extendedset.intset.ConciseSet; diff --git a/src/test/java/org/qortal/test/apps/LaunchExeWIthJvmOptions.java b/src/test/java/org/qortal/test/apps/LaunchExeWIthJvmOptions.java new file mode 100644 index 000000000..f64903a41 --- /dev/null +++ b/src/test/java/org/qortal/test/apps/LaunchExeWIthJvmOptions.java @@ -0,0 +1,63 @@ +package org.qortal.test.apps; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class LaunchExeWIthJvmOptions { + + private static final String JAR_FILENAME = "qortal.jar"; + private static final String WINDOWS_EXE_LAUNCHER = "qortal.exe"; + private static final String JAVA_TOOL_OPTIONS_NAME = "JAVA_TOOL_OPTIONS"; + private static final String JAVA_TOOL_OPTIONS_VALUE = "-XX:MaxRAMFraction=4"; + + public static void main(String[] args) { + String javaHome = System.getProperty("java.home"); + System.out.println(String.format("Java home: %s", javaHome)); + + Path javaBinary = Paths.get(javaHome, "bin", "java"); + System.out.println(String.format("Java binary: %s", javaBinary)); + + Path exeLauncher = Paths.get(WINDOWS_EXE_LAUNCHER); + System.out.println(String.format("Windows EXE launcher: %s", exeLauncher)); + + List javaCmd; + if (Files.exists(exeLauncher)) { + javaCmd = Arrays.asList(exeLauncher.toString()); + } else { + javaCmd = new ArrayList<>(); + // Java runtime binary itself + javaCmd.add(javaBinary.toString()); + + // JVM arguments + javaCmd.addAll(ManagementFactory.getRuntimeMXBean().getInputArguments()); + + // Call mainClass in JAR + javaCmd.addAll(Arrays.asList("-jar", JAR_FILENAME)); + + // Add saved command-line args + javaCmd.addAll(Arrays.asList(args)); + } + + try { + System.out.println(String.format("Restarting node with: %s", String.join(" ", javaCmd))); + + ProcessBuilder processBuilder = new ProcessBuilder(javaCmd); + + if (Files.exists(exeLauncher)) { + System.out.println(String.format("Setting env %s to %s", JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE)); + processBuilder.environment().put(JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE); + } + + processBuilder.start(); + } catch (IOException e) { + System.err.println(String.format("Failed to restart node (BAD): %s", e.getMessage())); + } + } + +} diff --git a/src/test/java/org/qortal/test/apps/VanityGen.java b/src/test/java/org/qortal/test/apps/VanityGen.java index f697087f8..2c22ea0bd 100644 --- a/src/test/java/org/qortal/test/apps/VanityGen.java +++ b/src/test/java/org/qortal/test/apps/VanityGen.java @@ -10,7 +10,6 @@ import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; import org.qortal.account.PrivateKeyAccount; import org.qortal.crypto.Crypto; -import org.qortal.utils.BIP39; import org.qortal.utils.Base58; import com.google.common.primitives.Bytes; @@ -44,15 +43,13 @@ public void run() { byte checksum = (byte) (hash[0] & 0xf0); byte[] entropy132 = Bytes.concat(entropy, new byte[] { checksum }); - String mnemonic = BIP39.encode(entropy132, "en"); - PrivateKeyAccount account = new PrivateKeyAccount(null, hash); if (!account.getAddress().startsWith(prefix)) continue; - System.out.println(String.format("Address: %s, public key: %s, private key: %s, mnemonic: %s", - account.getAddress(), Base58.encode(account.getPublicKey()), Base58.encode(hash), mnemonic)); + System.out.println(String.format("Address: %s, public key: %s, private key: %s", + account.getAddress(), Base58.encode(account.getPublicKey()), Base58.encode(hash))); System.out.flush(); } } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryCompressionTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryCompressionTests.java new file mode 100644 index 000000000..dd4820748 --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryCompressionTests.java @@ -0,0 +1,182 @@ +package org.qortal.test.arbitrary; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.arbitrary.ArbitraryDataDigest; +import org.qortal.crypto.Crypto; +import org.qortal.repository.DataException; +import org.qortal.test.common.Common; +import org.qortal.utils.ZipUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Random; + +import static org.junit.Assert.*; + +public class ArbitraryCompressionTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testZipSingleFile() throws IOException, InterruptedException { + String enclosingFolderName = "data"; + Path inputFile = Files.createTempFile("inputFile", null); + Path outputDirectory = Files.createTempDirectory("outputDirectory"); + Path outputFile = Paths.get(outputDirectory.toString(), enclosingFolderName); + inputFile.toFile().deleteOnExit(); + outputDirectory.toFile().deleteOnExit(); + + // Write random data to the input file + byte[] data = new byte[1024]; + new Random().nextBytes(data); + Files.write(inputFile, data, StandardOpenOption.CREATE); + + assertTrue(Files.exists(inputFile)); + assertFalse(Files.exists(outputFile)); + + // Zip... + ZipUtils.zip(inputFile.toString(), outputFile.toString(), enclosingFolderName); + + assertTrue(Files.exists(inputFile)); + assertTrue(Files.exists(outputFile)); + + // Ensure zipped file's hash differs from the original + assertFalse(Arrays.equals(Crypto.digest(inputFile.toFile()), Crypto.digest(outputFile.toFile()))); + + // Create paths for unzipping + Path unzippedDirectory = Files.createTempDirectory("unzippedDirectory"); + // Single file data is unzipped directly, without an enclosing folder. Original name is maintained. + Path unzippedFile = Paths.get(unzippedDirectory.toString(), enclosingFolderName, inputFile.getFileName().toString()); + unzippedDirectory.toFile().deleteOnExit(); + assertFalse(Files.exists(unzippedFile)); + + // Now unzip... + ZipUtils.unzip(outputFile.toString(), unzippedDirectory.toString()); + + // Ensure resulting file exists + assertTrue(Files.exists(unzippedFile)); + + // And make sure it matches the original input file + assertTrue(Arrays.equals(Crypto.digest(inputFile.toFile()), Crypto.digest(unzippedFile.toFile()))); + } + + @Test + public void testZipDirectoryWithSingleFile() throws IOException, InterruptedException, DataException { + String enclosingFolderName = "data"; + Path inputDirectory = Files.createTempDirectory("inputDirectory"); + Path outputDirectory = Files.createTempDirectory("outputDirectory"); + Path outputFile = Paths.get(outputDirectory.toString(), enclosingFolderName); + inputDirectory.toFile().deleteOnExit(); + outputDirectory.toFile().deleteOnExit(); + + Path inputFile = Paths.get(inputDirectory.toString(), "file"); + + // Write random data to a file + byte[] data = new byte[1024]; + new Random().nextBytes(data); + Files.write(inputFile, data, StandardOpenOption.CREATE); + + assertTrue(Files.exists(inputDirectory)); + assertTrue(Files.exists(inputFile)); + assertFalse(Files.exists(outputFile)); + + // Zip... + ZipUtils.zip(inputDirectory.toString(), outputFile.toString(), enclosingFolderName); + + assertTrue(Files.exists(inputDirectory)); + assertTrue(Files.exists(outputFile)); + + // Create paths for unzipping + Path unzippedDirectory = Files.createTempDirectory("unzippedDirectory"); + unzippedDirectory.toFile().deleteOnExit(); + Path unzippedFile = Paths.get(unzippedDirectory.toString(), enclosingFolderName, "file"); + assertFalse(Files.exists(unzippedFile)); + + // Now unzip... + ZipUtils.unzip(outputFile.toString(), unzippedDirectory.toString()); + + // Ensure resulting file exists + assertTrue(Files.exists(unzippedFile)); + + // And make sure they match the original input files + assertTrue(Arrays.equals(Crypto.digest(inputFile.toFile()), Crypto.digest(unzippedFile.toFile()))); + + // Unzipped files are placed within a folder named by the supplied enclosingFolderName + Path unzippedInnerDirectory = Paths.get(unzippedDirectory.toString(), enclosingFolderName); + + // Finally, make sure the directory digests match + ArbitraryDataDigest inputDirectoryDigest = new ArbitraryDataDigest(inputDirectory); + inputDirectoryDigest.compute(); + ArbitraryDataDigest unzippedDirectoryDigest = new ArbitraryDataDigest(unzippedInnerDirectory); + unzippedDirectoryDigest.compute(); + assertEquals(inputDirectoryDigest.getHash58(), unzippedDirectoryDigest.getHash58()); + } + + @Test + public void testZipMultipleFiles() throws IOException, InterruptedException, DataException { + String enclosingFolderName = "data"; + Path inputDirectory = Files.createTempDirectory("inputDirectory"); + Path outputDirectory = Files.createTempDirectory("outputDirectory"); + Path outputFile = Paths.get(outputDirectory.toString(), enclosingFolderName); + inputDirectory.toFile().deleteOnExit(); + outputDirectory.toFile().deleteOnExit(); + + Path inputFile1 = Paths.get(inputDirectory.toString(), "file1"); + Path inputFile2 = Paths.get(inputDirectory.toString(), "file2"); + + // Write random data to some files + byte[] data = new byte[1024]; + new Random().nextBytes(data); + Files.write(inputFile1, data, StandardOpenOption.CREATE); + Files.write(inputFile2, data, StandardOpenOption.CREATE); + + assertTrue(Files.exists(inputDirectory)); + assertTrue(Files.exists(inputFile1)); + assertTrue(Files.exists(inputFile2)); + assertFalse(Files.exists(outputFile)); + + // Zip... + ZipUtils.zip(inputDirectory.toString(), outputFile.toString(), enclosingFolderName); + + assertTrue(Files.exists(inputDirectory)); + assertTrue(Files.exists(outputFile)); + + // Create paths for unzipping + Path unzippedDirectory = Files.createTempDirectory("unzippedDirectory"); + unzippedDirectory.toFile().deleteOnExit(); + Path unzippedFile1 = Paths.get(unzippedDirectory.toString(), enclosingFolderName, "file1"); + Path unzippedFile2 = Paths.get(unzippedDirectory.toString(), enclosingFolderName, "file2"); + assertFalse(Files.exists(unzippedFile1)); + assertFalse(Files.exists(unzippedFile2)); + + // Now unzip... + ZipUtils.unzip(outputFile.toString(), unzippedDirectory.toString()); + + // Ensure resulting files exist + assertTrue(Files.exists(unzippedFile1)); + assertTrue(Files.exists(unzippedFile2)); + + // And make sure they match the original input files + assertTrue(Arrays.equals(Crypto.digest(inputFile1.toFile()), Crypto.digest(unzippedFile1.toFile()))); + assertTrue(Arrays.equals(Crypto.digest(inputFile2.toFile()), Crypto.digest(unzippedFile2.toFile()))); + + // Unzipped files are placed within a folder named by the supplied enclosingFolderName + Path unzippedInnerDirectory = Paths.get(unzippedDirectory.toString(), enclosingFolderName); + + // Finally, make sure the directory digests match + ArbitraryDataDigest inputDirectoryDigest = new ArbitraryDataDigest(inputDirectory); + inputDirectoryDigest.compute(); + ArbitraryDataDigest unzippedDirectoryDigest = new ArbitraryDataDigest(unzippedInnerDirectory); + unzippedDirectoryDigest.compute(); + assertEquals(inputDirectoryDigest.getHash58(), unzippedDirectoryDigest.getHash58()); + } + +} diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java new file mode 100644 index 000000000..8ef04b270 --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java @@ -0,0 +1,61 @@ +package org.qortal.test.arbitrary; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.arbitrary.ArbitraryDataDigest; +import org.qortal.repository.DataException; +import org.qortal.test.common.Common; + +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.UUID; + +import static org.junit.Assert.*; + +public class ArbitraryDataDigestTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testDirectoryDigest() throws IOException, DataException { + Path dataPath = Paths.get("src/test/resources/arbitrary/demo1"); + String expectedHash58 = "DKyMuonWKoneJqiVHgw26Vk1ytrZG9PGsE9xfBg3GKDp"; + + // Ensure directory exists + assertTrue(dataPath.toFile().exists()); + assertTrue(dataPath.toFile().isDirectory()); + + // Compute a hash + ArbitraryDataDigest digest = new ArbitraryDataDigest(dataPath); + digest.compute(); + assertEquals(expectedHash58, digest.getHash58()); + + // Write a random file to .qortal/cache to ensure it isn't being included in the digest function + // We exclude all .qortal files from the digest since they can be different with each build, and + // we only care about the actual user files + Path cachePath = Paths.get(dataPath.toString(), ".qortal", "cache"); + Files.createDirectories(cachePath.getParent()); + FileWriter fileWriter = new FileWriter(cachePath.toString()); + fileWriter.append(UUID.randomUUID().toString()); + fileWriter.close(); + + // Recompute the hash + digest = new ArbitraryDataDigest(dataPath); + digest.compute(); + assertEquals(expectedHash58, digest.getHash58()); + + // Now compute the hash 100 more times to ensure it's always the same + for (int i=0; i<100; i++) { + digest = new ArbitraryDataDigest(dataPath); + digest.compute(); + assertEquals(expectedHash58, digest.getHash58()); + } + } + +} diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java new file mode 100644 index 000000000..aabbe502d --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java @@ -0,0 +1,77 @@ +package org.qortal.test.arbitrary; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.repository.DataException; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.test.common.Common; + +import java.util.Random; + +import static org.junit.Assert.*; + +public class ArbitraryDataFileTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testSplitAndJoin() throws DataException { + String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; + ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes(), null); + assertTrue(arbitraryDataFile.exists()); + assertEquals(62, arbitraryDataFile.size()); + assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58()); + + // Split into 7 chunks, each 10 bytes long + arbitraryDataFile.split(10); + assertEquals(7, arbitraryDataFile.chunkCount()); + + // Delete the original file + arbitraryDataFile.delete(); + assertFalse(arbitraryDataFile.exists()); + assertEquals(0, arbitraryDataFile.size()); + + // Now rebuild the original file from the chunks + assertEquals(7, arbitraryDataFile.chunkCount()); + arbitraryDataFile.join(); + + // Validate that the original file is intact + assertTrue(arbitraryDataFile.exists()); + assertEquals(62, arbitraryDataFile.size()); + assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58()); + } + + @Test + public void testSplitAndJoinWithLargeFiles() throws DataException { + int fileSize = (int) (5.5f * 1024 * 1024); // 5.5MiB + byte[] randomData = new byte[fileSize]; + new Random().nextBytes(randomData); // No need for SecureRandom here + + ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData, null); + assertTrue(arbitraryDataFile.exists()); + assertEquals(fileSize, arbitraryDataFile.size()); + String originalFileDigest = arbitraryDataFile.digest58(); + + // Split into chunks using 1MiB chunk size + arbitraryDataFile.split(1 * 1024 * 1024); + assertEquals(6, arbitraryDataFile.chunkCount()); + + // Delete the original file + arbitraryDataFile.delete(); + assertFalse(arbitraryDataFile.exists()); + assertEquals(0, arbitraryDataFile.size()); + + // Now rebuild the original file from the chunks + assertEquals(6, arbitraryDataFile.chunkCount()); + arbitraryDataFile.join(); + + // Validate that the original file is intact + assertTrue(arbitraryDataFile.exists()); + assertEquals(fileSize, arbitraryDataFile.size()); + assertEquals(originalFileDigest, arbitraryDataFile.digest58()); + } + +} diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataMergeTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataMergeTests.java new file mode 100644 index 000000000..a366ef12c --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataMergeTests.java @@ -0,0 +1,445 @@ +package org.qortal.test.arbitrary; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.arbitrary.ArbitraryDataCombiner; +import org.qortal.arbitrary.ArbitraryDataCreatePatch; +import org.qortal.arbitrary.ArbitraryDataDigest; +import org.qortal.crypto.Crypto; +import org.qortal.repository.DataException; +import org.qortal.test.common.ArbitraryUtils; +import org.qortal.test.common.Common; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Objects; +import java.util.Random; + +import static org.junit.Assert.*; + +public class ArbitraryDataMergeTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testCreateAndMergePatch() throws IOException, DataException { + Path path1 = Paths.get("src/test/resources/arbitrary/demo1"); + Path path2 = Paths.get("src/test/resources/arbitrary/demo2"); + + // Generate random signature for the purposes of validation + byte[] signature = new byte[32]; + new Random().nextBytes(signature); + + // Create a patch using the differences in path2 compared with path1 + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(path1, path2, signature); + patch.create(); + Path patchPath = patch.getFinalPath(); + assertTrue(Files.exists(patchPath)); + + // Check that lorem1, 2, 4, and 5 exist + assertTrue(Files.exists(Paths.get(patchPath.toString(), "lorem1.txt"))); + assertTrue(Files.exists(Paths.get(patchPath.toString(), "lorem2.txt"))); + assertTrue(Files.exists(Paths.get(patchPath.toString(), "dir1", "lorem4.txt"))); + assertTrue(Files.exists(Paths.get(patchPath.toString(), "dir1", "dir2", "lorem5.txt"))); + + // Ensure that lorem3 doesn't exist, as this file is identical in the original paths + assertFalse(Files.exists(Paths.get(patchPath.toString(), "lorem3.txt"))); + + // Ensure that the patch files differ from the first path (except for lorem3, which is missing) + assertFalse(Arrays.equals( + Crypto.digest(Paths.get(path1.toString(), "lorem1.txt").toFile()), + Crypto.digest(Paths.get(patchPath.toString(), "lorem1.txt").toFile()) + )); + assertFalse(Arrays.equals( + Crypto.digest(Paths.get(path1.toString(), "lorem2.txt").toFile()), + Crypto.digest(Paths.get(patchPath.toString(), "lorem2.txt").toFile()) + )); + assertFalse(Arrays.equals( + Crypto.digest(Paths.get(path1.toString(), "dir1", "lorem4.txt").toFile()), + Crypto.digest(Paths.get(patchPath.toString(), "dir1", "lorem4.txt").toFile()) + )); + assertFalse(Arrays.equals( + Crypto.digest(Paths.get(path1.toString(), "dir1", "dir2", "lorem5.txt").toFile()), + Crypto.digest(Paths.get(patchPath.toString(), "dir1", "dir2", "lorem5.txt").toFile()) + )); + + // Ensure that patch files 1 and 4 differ from the original files + assertFalse(Arrays.equals( + Crypto.digest(Paths.get(path2.toString(), "lorem1.txt").toFile()), + Crypto.digest(Paths.get(patchPath.toString(), "lorem1.txt").toFile()) + )); + assertFalse(Arrays.equals( + Crypto.digest(Paths.get(path2.toString(), "dir1", "lorem4.txt").toFile()), + Crypto.digest(Paths.get(patchPath.toString(), "dir1", "lorem4.txt").toFile()) + )); + + // Files 2 and 5 should match the original files, because their contents were + // too small to create a patch file smaller than the original file + assertArrayEquals( + Crypto.digest(Paths.get(path2.toString(), "lorem2.txt").toFile()), + Crypto.digest(Paths.get(patchPath.toString(), "lorem2.txt").toFile()) + ); + assertArrayEquals( + Crypto.digest(Paths.get(path2.toString(), "dir1", "dir2", "lorem5.txt").toFile()), + Crypto.digest(Paths.get(patchPath.toString(), "dir1", "dir2", "lorem5.txt").toFile()) + ); + + // Now merge the patch with the original path + ArbitraryDataCombiner combiner = new ArbitraryDataCombiner(path1, patchPath, signature); + combiner.setShouldValidateHashes(true); + combiner.combine(); + Path finalPath = combiner.getFinalPath(); + + // Ensure that all files exist in the final path (including lorem3) + assertTrue(Files.exists(Paths.get(finalPath.toString(), "lorem1.txt"))); + assertTrue(Files.exists(Paths.get(finalPath.toString(), "lorem2.txt"))); + assertTrue(Files.exists(Paths.get(finalPath.toString(), "lorem3.txt"))); + assertTrue(Files.exists(Paths.get(finalPath.toString(), "dir1", "lorem4.txt"))); + assertTrue(Files.exists(Paths.get(finalPath.toString(), "dir1", "dir2", "lorem5.txt"))); + + // Ensure that the files match those in path2 exactly + assertArrayEquals( + Crypto.digest(Paths.get(finalPath.toString(), "lorem1.txt").toFile()), + Crypto.digest(Paths.get(path2.toString(), "lorem1.txt").toFile()) + ); + assertArrayEquals( + Crypto.digest(Paths.get(finalPath.toString(), "lorem2.txt").toFile()), + Crypto.digest(Paths.get(path2.toString(), "lorem2.txt").toFile()) + ); + assertArrayEquals( + Crypto.digest(Paths.get(finalPath.toString(), "lorem3.txt").toFile()), + Crypto.digest(Paths.get(path2.toString(), "lorem3.txt").toFile()) + ); + assertArrayEquals( + Crypto.digest(Paths.get(finalPath.toString(), "dir1", "lorem4.txt").toFile()), + Crypto.digest(Paths.get(path2.toString(), "dir1", "lorem4.txt").toFile()) + ); + assertArrayEquals( + Crypto.digest(Paths.get(finalPath.toString(), "dir1", "dir2", "lorem5.txt").toFile()), + Crypto.digest(Paths.get(path2.toString(), "dir1", "dir2", "lorem5.txt").toFile()) + ); + + // Also check that the directory digests match + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(path2); + path2Digest.compute(); + ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath); + finalPathDigest.compute(); + assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58()); + } + + @Test + public void testIdenticalPaths() throws IOException, DataException { + Path path = Paths.get("src/test/resources/arbitrary/demo1"); + + // Create a patch from two identical paths + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(path, path, new byte[16]); + + // Ensure that an exception is thrown due to matching states + try { + patch.create(); + fail("Creating patch should fail due to matching states"); + + } catch (DataException expectedException) { + assertEquals("Current state matches previous state. Nothing to do.", expectedException.getMessage()); + } + + } + + @Test + public void testMergeBinaryFiles() throws IOException, DataException { + // Create two files in random temp directories + Path tempDir1 = Files.createTempDirectory("testMergeBinaryFiles1"); + Path tempDir2 = Files.createTempDirectory("testMergeBinaryFiles2"); + File file1 = new File(Paths.get(tempDir1.toString(), "file.bin").toString()); + File file2 = new File(Paths.get(tempDir2.toString(), "file.bin").toString()); + file1.deleteOnExit(); + file2.deleteOnExit(); + + // Write random data to the first file + byte[] initialData = new byte[1024]; + new Random().nextBytes(initialData); + Files.write(file1.toPath(), initialData); + byte[] file1Digest = Crypto.digest(file1); + + // Write slightly modified data to the second file (bytes 100-116 are zeroed out) + byte[] updatedData = Arrays.copyOf(initialData, initialData.length); + final ByteBuffer byteBuffer = ByteBuffer.wrap(updatedData); + byteBuffer.position(100); + byteBuffer.put(new byte[16]); + updatedData = byteBuffer.array(); + Files.write(file2.toPath(), updatedData); + byte[] file2Digest = Crypto.digest(file2); + + // Make sure the two arrays are different + assertFalse(Arrays.equals(initialData, updatedData)); + + // And double check that they are both 1024 bytes long + assertEquals(1024, initialData.length); + assertEquals(1024, updatedData.length); + + // Ensure both files exist + assertTrue(Files.exists(file1.toPath())); + assertTrue(Files.exists(file2.toPath())); + + // Generate random signature for the purposes of validation + byte[] signature = new byte[32]; + new Random().nextBytes(signature); + + // Create a patch from the two paths + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(tempDir1, tempDir2, signature); + patch.create(); + Path patchPath = patch.getFinalPath(); + assertTrue(Files.exists(patchPath)); + + // Check that the patch file exists + Path patchFilePath = Paths.get(patchPath.toString(), "file.bin"); + assertTrue(Files.exists(patchFilePath)); + byte[] patchDigest = Crypto.digest(patchFilePath.toFile()); + + // Ensure that the patch file matches file2 exactly + // This is because binary files cannot currently be patched, and so the complete file + // is included instead + assertArrayEquals(patchDigest, file2Digest); + + // Make sure that the patch file is different from file1 + assertFalse(Arrays.equals(patchDigest, file1Digest)); + + // Now merge the patch with the original path + ArbitraryDataCombiner combiner = new ArbitraryDataCombiner(tempDir1, patchPath, signature); + combiner.setShouldValidateHashes(true); + combiner.combine(); + Path finalPath = combiner.getFinalPath(); + + // Check that the directory digests match + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(tempDir2); + path2Digest.compute(); + ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath); + finalPathDigest.compute(); + assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58()); + } + + @Test + public void testMergeRandomStrings() throws IOException, DataException { + // Create two files in random temp directories + Path tempDir1 = Files.createTempDirectory("testMergeRandomStrings"); + Path tempDir2 = Files.createTempDirectory("testMergeRandomStrings"); + File file1 = new File(Paths.get(tempDir1.toString(), "file.txt").toString()); + File file2 = new File(Paths.get(tempDir2.toString(), "file.txt").toString()); + file1.deleteOnExit(); + file2.deleteOnExit(); + + // Write a random string to the first file + BufferedWriter file1Writer = new BufferedWriter(new FileWriter(file1)); + String initialString = ArbitraryUtils.generateRandomString(1024); + // Add a newline every 50 chars + initialString = initialString.replaceAll("(.{50})", "$1\n"); + file1Writer.write(initialString); + file1Writer.newLine(); + file1Writer.close(); + byte[] file1Digest = Crypto.digest(file1); + + // Write a slightly modified string to the second file + BufferedWriter file2Writer = new BufferedWriter(new FileWriter(file2)); + String updatedString = initialString.concat("-edit"); + file2Writer.write(updatedString); + file2Writer.newLine(); + file2Writer.close(); + byte[] file2Digest = Crypto.digest(file2); + + // Make sure the two strings are different + assertFalse(Objects.equals(initialString, updatedString)); + + // Ensure both files exist + assertTrue(Files.exists(file1.toPath())); + assertTrue(Files.exists(file2.toPath())); + + // Generate random signature for the purposes of validation + byte[] signature = new byte[32]; + new Random().nextBytes(signature); + + // Create a patch from the two paths + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(tempDir1, tempDir2, signature); + patch.create(); + Path patchPath = patch.getFinalPath(); + assertTrue(Files.exists(patchPath)); + + // Check that the patch file exists + Path patchFilePath = Paths.get(patchPath.toString(), "file.txt"); + assertTrue(Files.exists(patchFilePath)); + byte[] patchDigest = Crypto.digest(patchFilePath.toFile()); + + // Make sure that the patch file is different from file1 and file2 + assertFalse(Arrays.equals(patchDigest, file1Digest)); + assertFalse(Arrays.equals(patchDigest, file2Digest)); + + // Now merge the patch with the original path + ArbitraryDataCombiner combiner = new ArbitraryDataCombiner(tempDir1, patchPath, signature); + combiner.setShouldValidateHashes(true); + combiner.combine(); + Path finalPath = combiner.getFinalPath(); + + // Check that the directory digests match + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(tempDir2); + path2Digest.compute(); + ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath); + finalPathDigest.compute(); + assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58()); + + } + + @Test + public void testMergeRandomStringsWithoutTrailingNewlines() throws IOException, DataException { + // Create two files in random temp directories + Path tempDir1 = Files.createTempDirectory("testMergeRandomStrings"); + Path tempDir2 = Files.createTempDirectory("testMergeRandomStrings"); + File file1 = new File(Paths.get(tempDir1.toString(), "file.txt").toString()); + File file2 = new File(Paths.get(tempDir2.toString(), "file.txt").toString()); + file1.deleteOnExit(); + file2.deleteOnExit(); + + // Write a random string to the first file + BufferedWriter file1Writer = new BufferedWriter(new FileWriter(file1)); + String initialString = ArbitraryUtils.generateRandomString(1024); + // Add a newline every 50 chars + initialString = initialString.replaceAll("(.{50})", "$1\n"); + // Remove newline at end of string + initialString = initialString.stripTrailing(); + file1Writer.write(initialString); + // No newline + file1Writer.close(); + byte[] file1Digest = Crypto.digest(file1); + + // Write a slightly modified string to the second file + BufferedWriter file2Writer = new BufferedWriter(new FileWriter(file2)); + String updatedString = initialString.concat("-edit"); + file2Writer.write(updatedString); + // No newline + file2Writer.close(); + byte[] file2Digest = Crypto.digest(file2); + + // Make sure the two strings are different + assertFalse(Objects.equals(initialString, updatedString)); + + // Ensure both files exist + assertTrue(Files.exists(file1.toPath())); + assertTrue(Files.exists(file2.toPath())); + + // Generate random signature for the purposes of validation + byte[] signature = new byte[32]; + new Random().nextBytes(signature); + + // Create a patch from the two paths + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(tempDir1, tempDir2, signature); + patch.create(); + Path patchPath = patch.getFinalPath(); + assertTrue(Files.exists(patchPath)); + + // Check that the patch file exists + Path patchFilePath = Paths.get(patchPath.toString(), "file.txt"); + assertTrue(Files.exists(patchFilePath)); + byte[] patchDigest = Crypto.digest(patchFilePath.toFile()); + + // Make sure that the patch file is different from file1 + assertFalse(Arrays.equals(patchDigest, file1Digest)); + + // Make sure that the patch file is different from file2 + assertFalse(Arrays.equals(patchDigest, file2Digest)); + + // Now merge the patch with the original path + ArbitraryDataCombiner combiner = new ArbitraryDataCombiner(tempDir1, patchPath, signature); + combiner.setShouldValidateHashes(true); + combiner.combine(); + Path finalPath = combiner.getFinalPath(); + + // Check that the directory digests match + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(tempDir2); + path2Digest.compute(); + ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath); + finalPathDigest.compute(); + assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58()); + + } + + @Test + public void testMergeRandomLargeStrings() throws IOException, DataException { + // Create two files in random temp directories + Path tempDir1 = Files.createTempDirectory("testMergeRandomStrings"); + Path tempDir2 = Files.createTempDirectory("testMergeRandomStrings"); + File file1 = new File(Paths.get(tempDir1.toString(), "file.txt").toString()); + File file2 = new File(Paths.get(tempDir2.toString(), "file.txt").toString()); + file1.deleteOnExit(); + file2.deleteOnExit(); + + // Write a random string to the first file + BufferedWriter file1Writer = new BufferedWriter(new FileWriter(file1)); + String initialString = ArbitraryUtils.generateRandomString(110 * 1024); + // Add a newline every 50 chars + initialString = initialString.replaceAll("(.{50})", "$1\n"); + file1Writer.write(initialString); + file1Writer.newLine(); + file1Writer.close(); + byte[] file1Digest = Crypto.digest(file1); + + // Write a slightly modified string to the second file + BufferedWriter file2Writer = new BufferedWriter(new FileWriter(file2)); + String updatedString = initialString.concat("-edit"); + file2Writer.write(updatedString); + file2Writer.newLine(); + file2Writer.close(); + byte[] file2Digest = Crypto.digest(file2); + + // Make sure the two strings are different + assertFalse(Objects.equals(initialString, updatedString)); + + // Ensure both files exist + assertTrue(Files.exists(file1.toPath())); + assertTrue(Files.exists(file2.toPath())); + + // Generate random signature for the purposes of validation + byte[] signature = new byte[32]; + new Random().nextBytes(signature); + + // Create a patch from the two paths + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(tempDir1, tempDir2, signature); + patch.create(); + Path patchPath = patch.getFinalPath(); + assertTrue(Files.exists(patchPath)); + + // Check that the patch file exists + Path patchFilePath = Paths.get(patchPath.toString(), "file.txt"); + assertTrue(Files.exists(patchFilePath)); + byte[] patchDigest = Crypto.digest(patchFilePath.toFile()); + + // The patch file should be identical to file2 because the source files + // were over the maximum size limit for creating patches + assertArrayEquals(patchDigest, file2Digest); + + // Make sure that the patch file is different from file1 + assertFalse(Arrays.equals(patchDigest, file1Digest)); + + // Now merge the patch with the original path + ArbitraryDataCombiner combiner = new ArbitraryDataCombiner(tempDir1, patchPath, signature); + combiner.setShouldValidateHashes(true); + combiner.combine(); + Path finalPath = combiner.getFinalPath(); + + // Check that the directory digests match + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(tempDir2); + path2Digest.compute(); + ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath); + finalPathDigest.compute(); + assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58()); + + } + +} diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java new file mode 100644 index 000000000..0523963e0 --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java @@ -0,0 +1,202 @@ +package org.qortal.test.arbitrary; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.arbitrary.ArbitraryDataCleanupManager; +import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.RegisterNameTransactionData; +import org.qortal.list.ResourceListManager; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.test.common.ArbitraryUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.Assert.*; + +public class ArbitraryDataStorageCapacityTests extends Common { + + @Before + public void beforeTest() throws DataException, InterruptedException, IllegalAccessException { + Common.useDefaultSettings(); + this.deleteDataDirectories(); + this.deleteListsDirectory(); + + // Set difficulty to 1 to speed up the tests + FieldUtils.writeField(ArbitraryDataManager.getInstance(), "powDifficulty", 1, true); + } + + @After + public void afterTest() throws DataException { + this.deleteDataDirectories(); + this.deleteListsDirectory(); + ArbitraryDataStorageManager.getInstance().shutdown(); + } + + + @Test + public void testCalculateTotalStorageCapacity() { + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + double storageFullThreshold = 0.9; // 90% + Long now = NTP.getTime(); + assertNotNull("NTP time must be synced", now); + long expectedTotalStorageCapacity = Settings.getInstance().getMaxStorageCapacity(); + + // Capacity isn't initially calculated + assertNull(storageManager.getStorageCapacity()); + assertEquals(0L, storageManager.getTotalDirectorySize()); + assertFalse(storageManager.isStorageCapacityCalculated()); + + // We need to calculate the directory size because we haven't yet + assertTrue(storageManager.shouldCalculateDirectorySize(now)); + storageManager.calculateDirectorySize(now); + assertTrue(storageManager.isStorageCapacityCalculated()); + + // Storage capacity should equal the value specified in settings + assertNotNull(storageManager.getStorageCapacity()); + assertEquals(expectedTotalStorageCapacity, storageManager.getStorageCapacity().longValue()); + + // We shouldn't calculate storage capacity again so soon + now += 9 * 60 * 1000L; + assertFalse(storageManager.shouldCalculateDirectorySize(now)); + + // ... but after 10 minutes we should recalculate + now += 1 * 60 * 1000L + 1L; + assertTrue(storageManager.shouldCalculateDirectorySize(now)); + } + + @Test + public void testCalculateStorageCapacityPerName() { + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + ResourceListManager resourceListManager = ResourceListManager.getInstance(); + double storageFullThreshold = 0.9; // 90% + Long now = NTP.getTime(); + assertNotNull("NTP time must be synced", now); + + // Capacity isn't initially calculated + assertNull(storageManager.getStorageCapacity()); + assertEquals(0L, storageManager.getTotalDirectorySize()); + assertFalse(storageManager.isStorageCapacityCalculated()); + + // We need to calculate the total directory size because we haven't yet + assertTrue(storageManager.shouldCalculateDirectorySize(now)); + storageManager.calculateDirectorySize(now); + assertTrue(storageManager.isStorageCapacityCalculated()); + + // Storage capacity should initially equal the total + assertEquals(0, resourceListManager.getItemCountForList("followedNames")); + long totalStorageCapacity = storageManager.getStorageCapacityIncludingThreshold(storageFullThreshold); + assertEquals(totalStorageCapacity, storageManager.storageCapacityPerName(storageFullThreshold)); + + // Follow some names + assertTrue(resourceListManager.addToList("followedNames", "Test1", false)); + assertTrue(resourceListManager.addToList("followedNames", "Test2", false)); + assertTrue(resourceListManager.addToList("followedNames", "Test3", false)); + assertTrue(resourceListManager.addToList("followedNames", "Test4", false)); + + // Ensure the followed name count is correct + assertEquals(4, resourceListManager.getItemCountForList("followedNames")); + + // Storage space per name should be the total storage capacity divided by the number of names + long expectedStorageCapacityPerName = (long)(totalStorageCapacity / 4.0f); + assertEquals(expectedStorageCapacityPerName, storageManager.storageCapacityPerName(storageFullThreshold)); + } + + + private void deleteDataDirectories() { + // Delete data directory if exists + Path dataPath = Paths.get(Settings.getInstance().getDataPath()); + try { + FileUtils.deleteDirectory(dataPath.toFile()); + } catch (IOException e) { + + } + + // Delete temp data directory if exists + Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath()); + try { + FileUtils.deleteDirectory(tempDataPath.toFile()); + } catch (IOException e) { + + } + } + + @Test + public void testDeleteRandomFilesForName() throws DataException, IOException, InterruptedException, IllegalAccessException { + try (final Repository repository = RepositoryManager.getRepository()) { + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + int chunkSize = 100; + int dataLength = 900; // Actual data length will be longer due to encryption + + // Set originalCopyIndicatorFileEnabled to false, otherwise nothing will be deleted as it all originates from this node + FieldUtils.writeField(Settings.getInstance(), "originalCopyIndicatorFileEnabled", false, true); + + // Alice hosts some data (with 10 chunks) + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String aliceName = "alice"; + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), aliceName, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + Path alicePath = ArbitraryUtils.generateRandomDataPath(dataLength); + ArbitraryDataFile aliceArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(alice.getPublicKey()), alicePath, aliceName, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize); + + // Bob hosts some data too (also with 10 chunks) + PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); + String bobName = "bob"; + transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, bob); + Path bobPath = ArbitraryUtils.generateRandomDataPath(dataLength); + ArbitraryDataFile bobArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(bob.getPublicKey()), bobPath, bobName, identifier, ArbitraryTransactionData.Method.PUT, service, bob, chunkSize); + + // All 20 chunks should exist + assertEquals(10, aliceArbitraryDataFile.chunkCount()); + assertTrue(aliceArbitraryDataFile.allChunksExist()); + assertEquals(10, bobArbitraryDataFile.chunkCount()); + assertTrue(bobArbitraryDataFile.allChunksExist()); + + // Now pretend that Bob has reached his storage limit - this should delete random files + // Run it 10 times to remove the likelihood of the randomizer always picking Alice's files + for (int i=0; i<10; i++) { + ArbitraryDataCleanupManager.getInstance().storageLimitReachedForName(repository, bobName); + } + + // Alice should still have all chunks + assertTrue(aliceArbitraryDataFile.allChunksExist()); + + // Bob should be missing some chunks + assertFalse(bobArbitraryDataFile.allChunksExist()); + + } + } + + private void deleteListsDirectory() { + // Delete lists directory if exists + Path listsPath = Paths.get(Settings.getInstance().getListsPath()); + try { + FileUtils.deleteDirectory(listsPath.toFile()); + } catch (IOException e) { + + } + } + +} diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java new file mode 100644 index 000000000..dd132eb00 --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java @@ -0,0 +1,285 @@ +package org.qortal.test.arbitrary; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.arbitrary.ArbitraryDataTransactionBuilder; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; +import org.qortal.controller.arbitrary.ArbitraryDataStorageManager.StoragePolicy; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.Method; +import org.qortal.data.transaction.RegisterNameTransactionData; +import org.qortal.list.ResourceListManager; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.Assert.*; + +public class ArbitraryDataStoragePolicyTests extends Common { + + @Before + public void beforeTest() throws DataException, InterruptedException { + Common.useDefaultSettings(); + this.deleteDataDirectories(); + this.deleteListsDirectory(); + ArbitraryDataStorageManager.getInstance().start(); + + // Wait for storage space to be calculated + while (!ArbitraryDataStorageManager.getInstance().isStorageCapacityCalculated()) { + Thread.sleep(100L); + } + } + + @After + public void afterTest() throws DataException { + this.deleteDataDirectories(); + this.deleteListsDirectory(); + ArbitraryDataStorageManager.getInstance().shutdown(); + } + + @Test + public void testFollowedAndViewed() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "Test"; + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create transaction + ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name); + + // Add name to followed list + assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); + + // We should store and pre-fetch data for this transaction + assertEquals(StoragePolicy.FOLLOWED_OR_VIEWED, Settings.getInstance().getStoragePolicy()); + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + + // Now unfollow the name + assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); + + // We should store but not pre-fetch data for this transaction + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + } + } + + @Test + public void testFollowedOnly() throws DataException, IllegalAccessException { + try (final Repository repository = RepositoryManager.getRepository()) { + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "Test"; + + // Set the storage policy to "FOLLOWED" + FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "FOLLOWED", true); + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create transaction + ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name); + + // Add name to followed list + assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); + + // We should store and pre-fetch data for this transaction + assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy()); + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + + // Now unfollow the name + assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); + + // We shouldn't store or pre-fetch data for this transaction + assertFalse(storageManager.canStoreData(arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + } + } + + @Test + public void testViewedOnly() throws DataException, IllegalAccessException { + try (final Repository repository = RepositoryManager.getRepository()) { + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "Test"; + + // Set the storage policy to "VIEWED" + FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "VIEWED", true); + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create transaction + ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name); + + // Add name to followed list + assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); + + // We should store but not pre-fetch data for this transaction + assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy()); + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + + // Now unfollow the name + assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); + + // We should store but not pre-fetch data for this transaction + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + } + } + + @Test + public void testAll() throws DataException, IllegalAccessException { + try (final Repository repository = RepositoryManager.getRepository()) { + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "Test"; + + // Set the storage policy to "ALL" + FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "ALL", true); + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create transaction + ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name); + + // Add name to followed list + assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); + + // We should store and pre-fetch data for this transaction + assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy()); + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + + // Now unfollow the name + assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); + + // We should store and pre-fetch data for this transaction + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + } + } + + @Test + public void testNone() throws DataException, IllegalAccessException { + try (final Repository repository = RepositoryManager.getRepository()) { + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "Test"; + + // Set the storage policy to "NONE" + FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "NONE", true); + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create transaction + ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name); + + // Add name to followed list + assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); + + // We shouldn't store or pre-fetch data for this transaction + assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy()); + assertFalse(storageManager.canStoreData(arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + + // Now unfollow the name + assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); + + // We shouldn't store or pre-fetch data for this transaction + assertFalse(storageManager.canStoreData(arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + } + } + + @Test + public void testTransactionWithoutName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = null; + + // Create transaction + ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name); + + // We should store but not pre-fetch data for this transaction + assertTrue(storageManager.canStoreData(transactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); + } + } + + private ArbitraryTransactionData createTxnWithName(Repository repository, PrivateKeyAccount acc, String name) throws DataException { + String publicKey58 = Base58.encode(acc.getPublicKey()); + Path path = Paths.get("src/test/resources/arbitrary/demo1"); + + ArbitraryDataTransactionBuilder txnBuilder = new ArbitraryDataTransactionBuilder( + repository, publicKey58, path, name, Method.PUT, Service.ARBITRARY_DATA, null); + + txnBuilder.build(); + ArbitraryTransactionData transactionData = txnBuilder.getArbitraryTransactionData(); + + return transactionData; + } + + private void deleteDataDirectories() { + // Delete data directory if exists + Path dataPath = Paths.get(Settings.getInstance().getDataPath()); + try { + FileUtils.deleteDirectory(dataPath.toFile()); + } catch (IOException e) { + + } + + // Delete temp data directory if exists + Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath()); + try { + FileUtils.deleteDirectory(tempDataPath.toFile()); + } catch (IOException e) { + + } + } + + private void deleteListsDirectory() { + // Delete lists directory if exists + Path listsPath = Paths.get(Settings.getInstance().getListsPath()); + try { + FileUtils.deleteDirectory(listsPath.toFile()); + } catch (IOException e) { + + } + } + +} diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java new file mode 100644 index 000000000..3a31e652c --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java @@ -0,0 +1,482 @@ +package org.qortal.test.arbitrary; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.arbitrary.ArbitraryDataDigest; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.arbitrary.ArbitraryDataReader; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.crypto.Crypto; +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.data.transaction.RegisterNameTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.test.common.ArbitraryUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Objects; + +import static org.junit.Assert.*; + +public class ArbitraryDataTests extends Common { + + @Before + public void beforeTest() throws DataException, IllegalAccessException { + Common.useDefaultSettings(); + + // Set difficulty to 1 to speed up the tests + FieldUtils.writeField(ArbitraryDataManager.getInstance(), "powDifficulty", 1, true); + } + + @Test + public void testCombineMultipleLayers() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = Paths.get("src/test/resources/arbitrary/demo1"); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); + + // Create PATCH transaction + Path path2 = Paths.get("src/test/resources/arbitrary/demo2"); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path2, name, identifier, Method.PATCH, service, alice); + + // Create another PATCH transaction + Path path3 = Paths.get("src/test/resources/arbitrary/demo3"); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path3, name, identifier, Method.PATCH, service, alice); + + // Now build the latest data state for this name + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); + arbitraryDataReader.loadSynchronously(true); + Path finalPath = arbitraryDataReader.getFilePath(); + + // Ensure it exists + assertTrue(Files.exists(finalPath)); + + // Its directory hash should match the hash of demo3 + ArbitraryDataDigest path3Digest = new ArbitraryDataDigest(path3); + path3Digest.compute(); + ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath); + finalPathDigest.compute(); + assertEquals(path3Digest.getHash58(), finalPathDigest.getHash58()); + + // .. and its directory hash should also match the one included in the metadata + ArbitraryDataMetadataPatch patchMetadata = new ArbitraryDataMetadataPatch(finalPath); + patchMetadata.read(); + assertArrayEquals(patchMetadata.getCurrentHash(), path3Digest.getHash()); + + } + } + + @Test + public void testPatchBeforePut() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + + // Create PATCH transaction, ensuring that an exception is thrown + try { + Path path1 = Paths.get("src/test/resources/arbitrary/demo1"); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PATCH, service, alice); + fail("Creating transaction should fail due to nonexistent PUT transaction"); + + } catch (DataException expectedException) { + assertEquals(String.format("Couldn't find PUT transaction for " + + "name %s, service %s and identifier ", name, service), expectedException.getMessage()); + } + + } + } + + @Test + public void testNameDoesNotExist() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + + // Ensure the name doesn't exist + assertNull(repository.getNameRepository().fromName(name)); + + // Create PUT transaction, ensuring that an exception is thrown + try { + Path path1 = Paths.get("src/test/resources/arbitrary/demo1"); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); + fail("Creating transaction should fail due to the name being unregistered"); + + } catch (DataException expectedException) { + assertEquals("Arbitrary transaction invalid: NAME_DOES_NOT_EXIST", expectedException.getMessage()); + } + } + } + + @Test + public void testUpdateResourceOwnedByAnotherCreator() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = Paths.get("src/test/resources/arbitrary/demo1"); + ArbitraryUtils.createAndMintTxn(repository, Base58.encode(alice.getPublicKey()), path1, name, identifier, Method.PUT, service, alice); + + // Bob attempts to update Alice's data + PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); + + // Create PATCH transaction, ensuring that an exception is thrown + try { + Path path2 = Paths.get("src/test/resources/arbitrary/demo2"); + ArbitraryUtils.createAndMintTxn(repository, Base58.encode(bob.getPublicKey()), path2, name, identifier, Method.PATCH, service, bob); + fail("Creating transaction should fail due to the name being registered to Alice instead of Bob"); + + } catch (DataException expectedException) { + assertEquals("Arbitrary transaction invalid: INVALID_NAME_OWNER", expectedException.getMessage()); + } + } + } + + @Test + public void testUpdateResource() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = Paths.get("src/test/resources/arbitrary/demo1"); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); + + // Now build the latest data state for this name + ArbitraryDataReader arbitraryDataReader1 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); + arbitraryDataReader1.loadSynchronously(true); + Path initialLayerPath = arbitraryDataReader1.getFilePath(); + ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath); + initialLayerDigest.compute(); + + // Create PATCH transaction + Path path2 = Paths.get("src/test/resources/arbitrary/demo2"); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path2, name, identifier, Method.PATCH, service, alice); + + // Rebuild the latest state + ArbitraryDataReader arbitraryDataReader2 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); + arbitraryDataReader2.loadSynchronously(false); + Path secondLayerPath = arbitraryDataReader2.getFilePath(); + ArbitraryDataDigest secondLayerDigest = new ArbitraryDataDigest(secondLayerPath); + secondLayerDigest.compute(); + + // Ensure that the second state is different to the first state + assertFalse(Arrays.equals(initialLayerDigest.getHash(), secondLayerDigest.getHash())); + + // Its directory hash should match the hash of demo2 + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(path2); + path2Digest.compute(); + assertEquals(path2Digest.getHash58(), secondLayerDigest.getHash58()); + } + } + + @Test + public void testIdentifier() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = "test_identifier"; + Service service = Service.ARBITRARY_DATA; + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = Paths.get("src/test/resources/arbitrary/demo1"); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); + + // Build the latest data state for this name, with a null identifier, ensuring that it fails + ArbitraryDataReader arbitraryDataReader1a = new ArbitraryDataReader(name, ResourceIdType.NAME, service, null); + try { + arbitraryDataReader1a.loadSynchronously(true); + fail("Loading data with null identifier should fail due to nonexistent PUT transaction"); + + } catch (DataException expectedException) { + assertEquals(String.format("Couldn't find PUT transaction for name %s, service %s " + + "and identifier ", name.toLowerCase(), service), expectedException.getMessage()); + } + + // Build the latest data state for this name, with a different identifier, ensuring that it fails + String differentIdentifier = "different_identifier"; + ArbitraryDataReader arbitraryDataReader1b = new ArbitraryDataReader(name, ResourceIdType.NAME, service, differentIdentifier); + try { + arbitraryDataReader1b.loadSynchronously(true); + fail("Loading data with incorrect identifier should fail due to nonexistent PUT transaction"); + + } catch (DataException expectedException) { + assertEquals(String.format("Couldn't find PUT transaction for name %s, service %s " + + "and identifier %s", name.toLowerCase(), service, differentIdentifier), expectedException.getMessage()); + } + + // Now build the latest data state for this name, with the correct identifier + ArbitraryDataReader arbitraryDataReader1c = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); + arbitraryDataReader1c.loadSynchronously(true); + Path initialLayerPath = arbitraryDataReader1c.getFilePath(); + ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath); + initialLayerDigest.compute(); + + // Create PATCH transaction + Path path2 = Paths.get("src/test/resources/arbitrary/demo2"); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path2, name, identifier, Method.PATCH, service, alice); + + // Rebuild the latest state + ArbitraryDataReader arbitraryDataReader2 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); + arbitraryDataReader2.loadSynchronously(false); + Path secondLayerPath = arbitraryDataReader2.getFilePath(); + ArbitraryDataDigest secondLayerDigest = new ArbitraryDataDigest(secondLayerPath); + secondLayerDigest.compute(); + + // Ensure that the second state is different to the first state + assertFalse(Arrays.equals(initialLayerDigest.getHash(), secondLayerDigest.getHash())); + + // Its directory hash should match the hash of demo2 + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(path2); + path2Digest.compute(); + assertEquals(path2Digest.getHash58(), secondLayerDigest.getHash58()); + } + } + + @Test + public void testBlankIdentifier() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = ""; // Blank, not null + Service service = Service.ARBITRARY_DATA; + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = Paths.get("src/test/resources/arbitrary/demo1"); + ArbitraryDataDigest path1Digest = new ArbitraryDataDigest(path1); + path1Digest.compute(); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); + + // Now build the latest data state for this name with a null identifier, ensuring that it succeeds and the data matches + ArbitraryDataReader arbitraryDataReader1a = new ArbitraryDataReader(name, ResourceIdType.NAME, service, null); + arbitraryDataReader1a.loadSynchronously(true); + Path initialLayerPath1a = arbitraryDataReader1a.getFilePath(); + ArbitraryDataDigest initialLayerDigest1a = new ArbitraryDataDigest(initialLayerPath1a); + initialLayerDigest1a.compute(); + assertEquals(path1Digest.getHash58(), initialLayerDigest1a.getHash58()); + + // It should also be accessible via a blank string, as we treat null and blank as the same thing + ArbitraryDataReader arbitraryDataReader1b = new ArbitraryDataReader(name, ResourceIdType.NAME, service, ""); + arbitraryDataReader1b.loadSynchronously(true); + Path initialLayerPath1b = arbitraryDataReader1b.getFilePath(); + ArbitraryDataDigest initialLayerDigest1b = new ArbitraryDataDigest(initialLayerPath1b); + initialLayerDigest1b.compute(); + assertEquals(path1Digest.getHash58(), initialLayerDigest1b.getHash58()); + + // Build the latest data state for this name, with a different identifier, ensuring that it fails + String differentIdentifier = "different_identifier"; + ArbitraryDataReader arbitraryDataReader1c = new ArbitraryDataReader(name, ResourceIdType.NAME, service, differentIdentifier); + try { + arbitraryDataReader1c.loadSynchronously(true); + fail("Loading data with incorrect identifier should fail due to nonexistent PUT transaction"); + + } catch (DataException expectedException) { + assertEquals(String.format("Couldn't find PUT transaction for name %s, service %s " + + "and identifier %s", name.toLowerCase(), service, differentIdentifier), expectedException.getMessage()); + } + } + } + + @Test + public void testSingleFile() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = "test1"; // Blank, not null + Service service = Service.DOCUMENT; // Can be anything for this test + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = Paths.get("src/test/resources/arbitrary/demo1/lorem1.txt"); + byte[] path1FileDigest = Crypto.digest(path1.toFile()); + ArbitraryDataDigest path1DirectoryDigest = new ArbitraryDataDigest(path1.getParent()); + path1DirectoryDigest.compute(); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); + + // Now build the latest data state for this name + ArbitraryDataReader arbitraryDataReader1 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); + arbitraryDataReader1.loadSynchronously(true); + Path builtFilePath = Paths.get(arbitraryDataReader1.getFilePath().toString(), path1.getFileName().toString()); + byte[] builtFileDigest = Crypto.digest(builtFilePath.toFile()); + + // Compare it against the hash of the original file + assertArrayEquals(builtFileDigest, path1FileDigest); + + // The directory digest won't match because the file is renamed to "data" + // We may need to find a way to retain the filename + ArbitraryDataDigest builtDirectoryDigest = new ArbitraryDataDigest(arbitraryDataReader1.getFilePath()); + builtDirectoryDigest.compute(); + assertFalse(Objects.equals(path1DirectoryDigest.getHash58(), builtDirectoryDigest.getHash58())); + } + } + + @Test + public void testOriginalCopyIndicatorFile() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = "test1"; // Blank, not null + Service service = Service.DOCUMENT; // Can be anything for this test + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = Paths.get("src/test/resources/arbitrary/demo1/lorem1.txt"); + ArbitraryDataDigest path1DirectoryDigest = new ArbitraryDataDigest(path1.getParent()); + path1DirectoryDigest.compute(); + ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); + + // Ensure that an ".original" file exists + Path parentPath = arbitraryDataFile.getFilePath().getParent(); + Path originalCopyIndicatorFile = Paths.get(parentPath.toString(), ".original"); + assertTrue(Files.exists(originalCopyIndicatorFile)); + } + } + + @Test + public void testOriginalCopyIndicatorFileDisabled() throws DataException, IOException, IllegalAccessException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = "test1"; // Blank, not null + Service service = Service.DOCUMENT; // Can be anything for this test + + // Set originalCopyIndicatorFileEnabled to false + FieldUtils.writeField(Settings.getInstance(), "originalCopyIndicatorFileEnabled", false, true); + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = Paths.get("src/test/resources/arbitrary/demo1/lorem1.txt"); + ArbitraryDataDigest path1DirectoryDigest = new ArbitraryDataDigest(path1.getParent()); + path1DirectoryDigest.compute(); + ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); + + // Ensure that an ".original" file exists + Path parentPath = arbitraryDataFile.getFilePath().getParent(); + Path originalCopyIndicatorFile = Paths.get(parentPath.toString(), ".original"); + assertFalse(Files.exists(originalCopyIndicatorFile)); + } + } + + @Test + public void testNameWithSpace() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "Test Name"; + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = Paths.get("src/test/resources/arbitrary/demo1"); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); + + // Create PATCH transaction + Path path2 = Paths.get("src/test/resources/arbitrary/demo2"); + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path2, name, identifier, Method.PATCH, service, alice); + + // Now build the latest data state for this name + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); + arbitraryDataReader.loadSynchronously(true); + Path finalPath = arbitraryDataReader.getFilePath(); + + // Ensure it exists + assertTrue(Files.exists(finalPath)); + + // Its directory hash should match the hash of demo2 + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(path2); + path2Digest.compute(); + ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath); + finalPathDigest.compute(); + assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58()); + + // .. and its directory hash should also match the one included in the metadata + ArbitraryDataMetadataPatch patchMetadata = new ArbitraryDataMetadataPatch(finalPath); + patchMetadata.read(); + assertArrayEquals(patchMetadata.getCurrentHash(), path2Digest.getHash()); + + } + } + +} diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryPeerTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryPeerTests.java new file mode 100644 index 000000000..b21713cba --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryPeerTests.java @@ -0,0 +1,155 @@ +package org.qortal.test.arbitrary; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.crypto.Crypto; +import org.qortal.data.network.ArbitraryPeerData; +import org.qortal.data.network.PeerData; +import org.qortal.network.Peer; +import org.qortal.network.PeerAddress; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.Common; +import org.qortal.utils.NTP; + +import java.util.Random; + +import static org.junit.Assert.*; + +public class ArbitraryPeerTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testSaveArbitraryPeerData() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + String peerAddress = "123.124.125.126:12392"; + String host = peerAddress.split(":")[0]; + + // Create random bytes to represent a signature + byte[] signature = new byte[64]; + new Random().nextBytes(signature); + + // Make sure we don't have an entry for this hash/peer combination + assertNull(repository.getArbitraryRepository().getArbitraryPeerDataForSignatureAndHost(signature, host)); + + // Now add this mapping to the db + Peer peer = new Peer(new PeerData(PeerAddress.fromString(peerAddress))); + ArbitraryPeerData arbitraryPeerData = new ArbitraryPeerData(signature, peer); + assertTrue(arbitraryPeerData.isPeerAddressValid()); + repository.getArbitraryRepository().save(arbitraryPeerData); + + // We should now have an entry for this hash/peer combination + ArbitraryPeerData retrievedArbitraryPeerData = repository.getArbitraryRepository() + .getArbitraryPeerDataForSignatureAndHost(signature, host); + assertNotNull(retrievedArbitraryPeerData); + + // .. and its data should match what was saved + assertArrayEquals(Crypto.digest(signature), retrievedArbitraryPeerData.getHash()); + assertEquals(peerAddress, retrievedArbitraryPeerData.getPeerAddress()); + + } + } + + @Test + public void testUpdateArbitraryPeerData() throws DataException, InterruptedException { + try (final Repository repository = RepositoryManager.getRepository()) { + + String peerAddress = "123.124.125.126:12392"; + String host = peerAddress.split(":")[0]; + + // Create random bytes to represent a signature + byte[] signature = new byte[64]; + new Random().nextBytes(signature); + + // Make sure we don't have an entry for this hash/peer combination + assertNull(repository.getArbitraryRepository().getArbitraryPeerDataForSignatureAndHost(signature, host)); + + // Now add this mapping to the db + Peer peer = new Peer(new PeerData(PeerAddress.fromString(peerAddress))); + ArbitraryPeerData arbitraryPeerData = new ArbitraryPeerData(signature, peer); + assertTrue(arbitraryPeerData.isPeerAddressValid()); + repository.getArbitraryRepository().save(arbitraryPeerData); + + // We should now have an entry for this hash/peer combination + ArbitraryPeerData retrievedArbitraryPeerData = repository.getArbitraryRepository() + .getArbitraryPeerDataForSignatureAndHost(signature, host); + assertNotNull(retrievedArbitraryPeerData); + + // .. and its data should match what was saved + assertArrayEquals(Crypto.digest(signature), retrievedArbitraryPeerData.getHash()); + assertEquals(peerAddress, retrievedArbitraryPeerData.getPeerAddress()); + + // All stats should be zero + assertEquals(Integer.valueOf(0), retrievedArbitraryPeerData.getSuccesses()); + assertEquals(Integer.valueOf(0), retrievedArbitraryPeerData.getFailures()); + assertEquals(Long.valueOf(0), retrievedArbitraryPeerData.getLastAttempted()); + assertEquals(Long.valueOf(0), retrievedArbitraryPeerData.getLastRetrieved()); + + // Now modify some values and re-save + retrievedArbitraryPeerData.incrementSuccesses(); retrievedArbitraryPeerData.incrementSuccesses(); // Twice + retrievedArbitraryPeerData.incrementFailures(); // Once + retrievedArbitraryPeerData.markAsAttempted(); + Thread.sleep(100); + retrievedArbitraryPeerData.markAsRetrieved(); + assertTrue(arbitraryPeerData.isPeerAddressValid()); + repository.getArbitraryRepository().save(retrievedArbitraryPeerData); + + // Retrieve data once again + ArbitraryPeerData updatedArbitraryPeerData = repository.getArbitraryRepository() + .getArbitraryPeerDataForSignatureAndHost(signature, host); + assertNotNull(updatedArbitraryPeerData); + + // Check the values + assertArrayEquals(Crypto.digest(signature), updatedArbitraryPeerData.getHash()); + assertEquals(peerAddress, updatedArbitraryPeerData.getPeerAddress()); + assertEquals(Integer.valueOf(2), updatedArbitraryPeerData.getSuccesses()); + assertEquals(Integer.valueOf(1), updatedArbitraryPeerData.getFailures()); + assertTrue(updatedArbitraryPeerData.getLastRetrieved().longValue() > 0L); + assertTrue(updatedArbitraryPeerData.getLastAttempted().longValue() > 0L); + assertTrue(updatedArbitraryPeerData.getLastRetrieved() > updatedArbitraryPeerData.getLastAttempted()); + assertTrue(NTP.getTime() - updatedArbitraryPeerData.getLastRetrieved() < 1000); + assertTrue(NTP.getTime() - updatedArbitraryPeerData.getLastAttempted() < 1000); + } + } + + @Test + public void testDuplicatePeerHost() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + String peerAddress1 = "123.124.125.126:12392"; + String peerAddress2 = "123.124.125.126:62392"; + String host1 = peerAddress1.split(":")[0]; + String host2 = peerAddress2.split(":")[0]; + + // Create random bytes to represent a signature + byte[] signature = new byte[64]; + new Random().nextBytes(signature); + + // Make sure we don't have an entry for these hash/peer combinations + assertNull(repository.getArbitraryRepository().getArbitraryPeerDataForSignatureAndHost(signature, host1)); + assertNull(repository.getArbitraryRepository().getArbitraryPeerDataForSignatureAndHost(signature, host2)); + + // Now add this mapping to the db + Peer peer = new Peer(new PeerData(PeerAddress.fromString(peerAddress1))); + ArbitraryPeerData arbitraryPeerData = new ArbitraryPeerData(signature, peer); + assertTrue(arbitraryPeerData.isPeerAddressValid()); + repository.getArbitraryRepository().save(arbitraryPeerData); + + // We should now have an entry for this hash/peer combination + ArbitraryPeerData retrievedArbitraryPeerData = repository.getArbitraryRepository() + .getArbitraryPeerDataForSignatureAndHost(signature, host1); + assertNotNull(retrievedArbitraryPeerData); + + // And we should also have an entry for the similar peerAddress string with a matching host + ArbitraryPeerData retrievedArbitraryPeerData2 = repository.getArbitraryRepository() + .getArbitraryPeerDataForSignatureAndHost(signature, host2); + assertNotNull(retrievedArbitraryPeerData2); + } + } +} diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java new file mode 100644 index 000000000..4db8bdc76 --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java @@ -0,0 +1,178 @@ +package org.qortal.test.arbitrary; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.arbitrary.misc.Service; +import org.qortal.arbitrary.misc.Service.ValidationResult; +import org.qortal.repository.DataException; +import org.qortal.test.common.Common; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Random; + +import static org.junit.Assert.*; + +public class ArbitraryServiceTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testDefaultValidation() throws IOException { + // We don't validate the ARBITRARY_DATA service specifically, so we can use it to test the default validation method + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write to temp path + Path path = Files.createTempFile("testDefaultValidation", null); + path.toFile().deleteOnExit(); + Files.write(path, data, StandardOpenOption.CREATE); + + Service service = Service.ARBITRARY_DATA; + assertFalse(service.isValidationRequired()); + // Test validation anyway to ensure that no exception is thrown + assertEquals(ValidationResult.OK, service.validate(path)); + } + + @Test + public void testValidateWebsite() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to several files in a temp path + Path path = Files.createTempDirectory("testValidateWebsite"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "index.html"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "data2"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "data3"), data, StandardOpenOption.CREATE); + + Service service = Service.WEBSITE; + assertTrue(service.isValidationRequired()); + + // There is an index file in the root + assertEquals(ValidationResult.OK, service.validate(path)); + } + + @Test + public void testValidateWebsiteWithoutIndexFile() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to several files in a temp path + Path path = Files.createTempDirectory("testValidateWebsiteWithoutIndexFile"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "data1.html"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "data2"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "data3"), data, StandardOpenOption.CREATE); + + Service service = Service.WEBSITE; + assertTrue(service.isValidationRequired()); + + // There is no index file in the root + assertEquals(ValidationResult.MISSING_INDEX_FILE, service.validate(path)); + } + + @Test + public void testValidateWebsiteWithoutIndexFileInRoot() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to several files in a temp path + Path path = Files.createTempDirectory("testValidateWebsiteWithoutIndexFileInRoot"); + path.toFile().deleteOnExit(); + Files.createDirectories(Paths.get(path.toString(), "directory")); + Files.write(Paths.get(path.toString(), "directory", "index.html"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "data2"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "data3"), data, StandardOpenOption.CREATE); + + Service service = Service.WEBSITE; + assertTrue(service.isValidationRequired()); + + // There is no index file in the root + assertEquals(ValidationResult.MISSING_INDEX_FILE, service.validate(path)); + } + + @Test + public void testValidQortalMetadata() throws IOException { + // Metadata is to describe an arbitrary resource (title, description, tags, etc) + String dataString = "{\"title\":\"Test Title\", \"description\":\"Test description\", \"tags\":[\"test\"]}"; + + // Write to temp path + Path path = Files.createTempFile("testValidQortalMetadata", null); + path.toFile().deleteOnExit(); + Files.write(path, dataString.getBytes(), StandardOpenOption.CREATE); + + Service service = Service.QORTAL_METADATA; + assertTrue(service.isValidationRequired()); + assertEquals(ValidationResult.OK, service.validate(path)); + } + + @Test + public void testQortalMetadataMissingKeys() throws IOException { + // Metadata is to describe an arbitrary resource (title, description, tags, etc) + String dataString = "{\"description\":\"Test description\", \"tags\":[\"test\"]}"; + + // Write to temp path + Path path = Files.createTempFile("testQortalMetadataMissingKeys", null); + path.toFile().deleteOnExit(); + Files.write(path, dataString.getBytes(), StandardOpenOption.CREATE); + + Service service = Service.QORTAL_METADATA; + assertTrue(service.isValidationRequired()); + assertEquals(ValidationResult.MISSING_KEYS, service.validate(path)); + } + + @Test + public void testQortalMetadataTooLarge() throws IOException { + // Metadata is to describe an arbitrary resource (title, description, tags, etc) + String dataString = "{\"title\":\"Test Title\", \"description\":\"Test description\", \"tags\":[\"test\"]}"; + + // Generate some large data to go along with it + int largeDataSize = 11*1024; // Larger than allowed 10kiB + byte[] largeData = new byte[largeDataSize]; + new Random().nextBytes(largeData); + + // Write to temp path + Path path = Files.createTempDirectory("testQortalMetadataTooLarge"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "data"), dataString.getBytes(), StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "large_data"), largeData, StandardOpenOption.CREATE); + + Service service = Service.QORTAL_METADATA; + assertTrue(service.isValidationRequired()); + assertEquals(ValidationResult.EXCEEDS_SIZE_LIMIT, service.validate(path)); + } + + @Test + public void testMultipleFileMetadata() throws IOException { + // Metadata is to describe an arbitrary resource (title, description, tags, etc) + String dataString = "{\"title\":\"Test Title\", \"description\":\"Test description\", \"tags\":[\"test\"]}"; + + // Generate some large data to go along with it + int otherDataSize = 1024; // Smaller than 10kiB limit + byte[] otherData = new byte[otherDataSize]; + new Random().nextBytes(otherData); + + // Write to temp path + Path path = Files.createTempDirectory("testMultipleFileMetadata"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "data"), dataString.getBytes(), StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "other_data"), otherData, StandardOpenOption.CREATE); + + Service service = Service.QORTAL_METADATA; + assertTrue(service.isValidationRequired()); + + // There are multiple files, so we don't know which one to parse as JSON + assertEquals(ValidationResult.MISSING_KEYS, service.validate(path)); + } + +} diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java new file mode 100644 index 000000000..4d82f1f2c --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java @@ -0,0 +1,79 @@ +package org.qortal.test.arbitrary; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.arbitrary.ArbitraryDataDigest; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.arbitrary.ArbitraryDataReader; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.RegisterNameTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.ArbitraryUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.junit.Assert.*; + +public class ArbitraryTransactionMetadataTests extends Common { + + @Before + public void beforeTest() throws DataException, IllegalAccessException { + Common.useDefaultSettings(); + + // Set difficulty to 1 to speed up the tests + FieldUtils.writeField(ArbitraryDataManager.getInstance(), "powDifficulty", 1, true); + } + + @Test + public void testMultipleChunks() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + int chunkSize = 100; + int dataLength = 900; // Actual data length will be longer due to encryption + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength); + ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize); + + // Check the chunk count is correct + assertEquals(10, arbitraryDataFile.chunkCount()); + + // Now build the latest data state for this name + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); + arbitraryDataReader.loadSynchronously(true); + Path initialLayerPath = arbitraryDataReader.getFilePath(); + ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath); + initialLayerDigest.compute(); + + // Its directory hash should match the original directory hash + ArbitraryDataDigest path1Digest = new ArbitraryDataDigest(path1); + path1Digest.compute(); + assertEquals(path1Digest.getHash58(), initialLayerDigest.getHash58()); + } + } + +} diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java new file mode 100644 index 000000000..866aa3389 --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java @@ -0,0 +1,84 @@ +package org.qortal.test.arbitrary; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.RegisterNameTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.ArbitraryUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.ArbitraryTransaction; +import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.junit.Assert.*; + +public class ArbitraryTransactionTests extends Common { + + @Before + public void beforeTest() throws DataException, IllegalAccessException { + Common.useDefaultSettings(); + } + + @Test + public void testDifficultyTooLow() throws IllegalAccessException, DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + int chunkSize = 100; + int dataLength = 900; // Actual data length will be longer due to encryption + + // Register the name to Alice + RegisterNameTransactionData registerNameTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + registerNameTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(registerNameTransactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, registerNameTransactionData, alice); + + // Set difficulty to 1 + FieldUtils.writeField(ArbitraryDataManager.getInstance(), "powDifficulty", 1, true); + + // Create PUT transaction + Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength); + ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize); + + // Check that nonce validation succeeds + byte[] signature = arbitraryDataFile.getSignature(); + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + ArbitraryTransaction transaction = new ArbitraryTransaction(repository, transactionData); + assertTrue(transaction.isSignatureValid()); + + // Increase difficulty to 15 + FieldUtils.writeField(ArbitraryDataManager.getInstance(), "powDifficulty", 15, true); + + // Make sure the nonce validation fails + // Note: there is a very tiny chance this could succeed due to being extremely lucky + // and finding a high difficulty nonce in the first couple of cycles. It will be rare + // enough that we shouldn't need to account for it. + assertFalse(transaction.isSignatureValid()); + + // Reduce difficulty back to 1, to double check + FieldUtils.writeField(ArbitraryDataManager.getInstance(), "powDifficulty", 1, true); + assertTrue(transaction.isSignatureValid()); + + } + + } + +} diff --git a/src/test/java/org/qortal/test/at/AtRepositoryTests.java b/src/test/java/org/qortal/test/at/AtRepositoryTests.java new file mode 100644 index 000000000..8ef4c7743 --- /dev/null +++ b/src/test/java/org/qortal/test/at/AtRepositoryTests.java @@ -0,0 +1,420 @@ +package org.qortal.test.at; + +import static org.junit.Assert.*; + +import java.nio.ByteBuffer; +import java.util.List; + +import org.ciyam.at.CompilationException; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.AtUtils; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; + +public class AtRepositoryTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testGetATStateAtHeightWithData() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + + Integer testHeight = 8; + ATStateData atStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight); + + assertEquals(testHeight, atStateData.getHeight()); + assertNotNull(atStateData.getStateData()); + } + } + + @Test + public void testGetATStateAtHeightWithoutData() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + + int maxHeight = 8; + Integer testHeight = maxHeight - 2; + + // Trim AT state data + repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().trimAtStates(2, maxHeight, 1000); + + ATStateData atStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight); + + assertEquals(testHeight, atStateData.getHeight()); + assertNull(atStateData.getStateData()); + } + } + + @Test + public void testGetLatestATStateWithData() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + int blockchainHeight = repository.getBlockRepository().getBlockchainHeight(); + + Integer testHeight = blockchainHeight; + ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); + + assertEquals(testHeight, atStateData.getHeight()); + assertNotNull(atStateData.getStateData()); + } + } + + @Test + public void testGetLatestATStatePostTrimming() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + int blockchainHeight = repository.getBlockRepository().getBlockchainHeight(); + + int maxHeight = blockchainHeight + 100; // more than latest block height + Integer testHeight = blockchainHeight; + + // Trim AT state data + repository.getATRepository().rebuildLatestAtStates(); + // COMMIT to check latest AT states persist / TEMPORARY table interaction + repository.saveChanges(); + + repository.getATRepository().trimAtStates(2, maxHeight, 1000); + + ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); + + assertEquals(testHeight, atStateData.getHeight()); + // We should always have the latest AT state data available + assertNotNull(atStateData.getStateData()); + } + } + + @Test + public void testOrphanTrimmedATStates() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + + int blockchainHeight = repository.getBlockRepository().getBlockchainHeight(); + int maxTrimHeight = blockchainHeight - 4; + Integer testHeight = maxTrimHeight + 1; + + // Trim AT state data + repository.getATRepository().rebuildLatestAtStates(); + repository.saveChanges(); + repository.getATRepository().trimAtStates(2, maxTrimHeight, 1000); + + // Orphan 3 blocks + // This leaves one more untrimmed block, so the latest AT state should be available + BlockUtils.orphanBlocks(repository, 3); + + ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); + assertEquals(testHeight, atStateData.getHeight()); + + // We should always have the latest AT state data available + assertNotNull(atStateData.getStateData()); + + // Orphan 1 more block + Exception exception = null; + try { + BlockUtils.orphanBlocks(repository, 1); + } catch (DataException e) { + exception = e; + } + + // Ensure that a DataException is thrown because there is no more AT states data available + assertNotNull(exception); + assertEquals(DataException.class, exception.getClass()); + assertEquals(String.format("Can't find previous AT state data for %s", atAddress), exception.getMessage()); + + // FUTURE: we may be able to retain unique AT states when trimming, to avoid this exception + // and allow orphaning back through blocks with trimmed AT states. + } + } + + @Test + public void testGetMatchingFinalATStatesWithoutDataValue() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + int blockchainHeight = repository.getBlockRepository().getBlockchainHeight(); + + Integer testHeight = blockchainHeight; + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + + byte[] codeHash = atData.getCodeHash(); + Boolean isFinished = Boolean.FALSE; + Integer dataByteOffset = null; + Long expectedValue = null; + Integer minimumFinalHeight = null; + Integer limit = null; + Integer offset = null; + Boolean reverse = null; + + List atStates = repository.getATRepository().getMatchingFinalATStates( + codeHash, + isFinished, + dataByteOffset, + expectedValue, + minimumFinalHeight, + limit, offset, reverse); + + assertEquals(false, atStates.isEmpty()); + assertEquals(1, atStates.size()); + + ATStateData atStateData = atStates.get(0); + assertEquals(testHeight, atStateData.getHeight()); + assertNotNull(atStateData.getStateData()); + } + } + + @Test + public void testGetMatchingFinalATStatesWithDataValue() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + int blockchainHeight = repository.getBlockRepository().getBlockchainHeight(); + + Integer testHeight = blockchainHeight; + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + + byte[] codeHash = atData.getCodeHash(); + Boolean isFinished = Boolean.FALSE; + Integer dataByteOffset = MachineState.HEADER_LENGTH + 0; + Long expectedValue = 0L; + Integer minimumFinalHeight = null; + Integer limit = null; + Integer offset = null; + Boolean reverse = null; + + List atStates = repository.getATRepository().getMatchingFinalATStates( + codeHash, + isFinished, + dataByteOffset, + expectedValue, + minimumFinalHeight, + limit, offset, reverse); + + assertEquals(false, atStates.isEmpty()); + assertEquals(1, atStates.size()); + + ATStateData atStateData = atStates.get(0); + assertEquals(testHeight, atStateData.getHeight()); + assertNotNull(atStateData.getStateData()); + } + } + + @Test + public void testGetBlockATStatesAtHeightWithData() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + + Integer testHeight = 8; + List atStates = repository.getATRepository().getBlockATStatesAtHeight(testHeight); + + assertEquals(false, atStates.isEmpty()); + assertEquals(1, atStates.size()); + + ATStateData atStateData = atStates.get(0); + assertEquals(testHeight, atStateData.getHeight()); + // getBlockATStatesAtHeight never returns actual AT state data anyway + assertNull(atStateData.getStateData()); + } + } + + @Test + public void testGetBlockATStatesAtHeightWithoutData() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + + int maxHeight = 8; + Integer testHeight = maxHeight - 2; + + // Trim AT state data + repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().trimAtStates(2, maxHeight, 1000); + + List atStates = repository.getATRepository().getBlockATStatesAtHeight(testHeight); + + assertEquals(false, atStates.isEmpty()); + assertEquals(1, atStates.size()); + + ATStateData atStateData = atStates.get(0); + assertEquals(testHeight, atStateData.getHeight()); + // getBlockATStatesAtHeight never returns actual AT state data anyway + assertNull(atStateData.getStateData()); + } + } + + @Test + public void testSaveATStateWithData() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + int blockchainHeight = repository.getBlockRepository().getBlockchainHeight(); + + Integer testHeight = blockchainHeight - 2; + ATStateData atStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight); + + assertEquals(testHeight, atStateData.getHeight()); + assertNotNull(atStateData.getStateData()); + + repository.getATRepository().save(atStateData); + + atStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight); + + assertEquals(testHeight, atStateData.getHeight()); + assertNotNull(atStateData.getStateData()); + } + } + + @Test + public void testSaveATStateWithoutData() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + int blockchainHeight = repository.getBlockRepository().getBlockchainHeight(); + + Integer testHeight = blockchainHeight - 2; + ATStateData atStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight); + + assertEquals(testHeight, atStateData.getHeight()); + assertNotNull(atStateData.getStateData()); + + // Clear data + ATStateData newAtStateData = new ATStateData(atStateData.getATAddress(), + atStateData.getHeight(), + /*StateData*/ null, + atStateData.getStateHash(), + atStateData.getFees(), + atStateData.isInitial(), + atStateData.getSleepUntilMessageTimestamp()); + repository.getATRepository().save(newAtStateData); + + atStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight); + + assertEquals(testHeight, atStateData.getHeight()); + assertNull(atStateData.getStateData()); + } + } +} diff --git a/src/test/java/org/qortal/test/at/GetMessageLengthTests.java b/src/test/java/org/qortal/test/at/GetMessageLengthTests.java new file mode 100644 index 000000000..e7a7bcc43 --- /dev/null +++ b/src/test/java/org/qortal/test/at/GetMessageLengthTests.java @@ -0,0 +1,220 @@ +package org.qortal.test.at; + +import static org.junit.Assert.*; + +import java.nio.ByteBuffer; +import java.util.Random; + +import org.ciyam.at.CompilationException; +import org.ciyam.at.FunctionCode; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.data.at.ATStateData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.AccountUtils; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.utils.BitTwiddling; + +public class GetMessageLengthTests extends Common { + + private static final Random RANDOM = new Random(); + + @Before + public void before() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testGetMessageLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + byte[] creationBytes = buildMessageLengthAT(); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Send messages with known length + checkMessageLength(repository, deployer, atAddress, 1); + checkMessageLength(repository, deployer, atAddress, 10); + checkMessageLength(repository, deployer, atAddress, 32); + checkMessageLength(repository, deployer, atAddress, 99); + + // Finally, send a payment instead and check returned length is -1 + AccountUtils.pay(repository, deployer, atAddress, 123L); + // Mint another block so AT can process payment + BlockUtils.mintBlock(repository); + + // Check AT result + ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); + byte[] stateData = atStateData.getStateData(); + + byte[] dataBytes = MachineState.extractDataBytes(stateData); + + long extractedLength = BitTwiddling.longFromBEBytes(dataBytes, 0); + + assertEquals(-1L, extractedLength); + } + } + + private void checkMessageLength(Repository repository, PrivateKeyAccount sender, String atAddress, int messageLength) throws DataException { + byte[] testMessage = new byte[messageLength]; + RANDOM.nextBytes(testMessage); + + sendMessage(repository, sender, testMessage, atAddress); + // Mint another block so AT can process message + BlockUtils.mintBlock(repository); + + // Check AT result + ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); + byte[] stateData = atStateData.getStateData(); + + byte[] dataBytes = MachineState.extractDataBytes(stateData); + + long extractedLength = BitTwiddling.longFromBEBytes(dataBytes, 0); + + assertEquals(messageLength, extractedLength); + } + + private byte[] buildMessageLengthAT() { + // Labels for data segment addresses + int addrCounter = 0; + + // Make result first for easier extraction + final int addrResult = addrCounter++; + final int addrLastTxTimestamp = addrCounter++; + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // Code labels + Integer labelCheckTx = null; + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(512); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxTimestamp)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message to AT */ + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, OpCode.calcOffset(codeByteBuffer, labelCheckTx))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckTx = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxTimestamp)); + // Save message length + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrResult)); + + // Stop and wait for next block (and hence more transactions) + codeByteBuffer.put(OpCode.STP_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile AT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "Test AT"; + String description = "Test AT"; + String atType = "Test"; + String tags = "TEST"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, messageTransactionData, sender); + + return messageTransaction; + } + +} diff --git a/src/test/java/org/qortal/test/at/GetNextTransactionTests.java b/src/test/java/org/qortal/test/at/GetNextTransactionTests.java new file mode 100644 index 000000000..c2eb9ede4 --- /dev/null +++ b/src/test/java/org/qortal/test/at/GetNextTransactionTests.java @@ -0,0 +1,274 @@ +package org.qortal.test.at; + +import static org.junit.Assert.*; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import org.ciyam.at.CompilationException; +import org.ciyam.at.FunctionCode; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.ciyam.at.Timestamp; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.block.Block; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.utils.BitTwiddling; + +public class GetNextTransactionTests extends Common { + + @Before + public void before() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testGetNextTransaction() throws DataException { + byte[] data = new byte[] { 0x44 }; + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + byte[] creationBytes = buildGetNextTransactionAT(); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + byte[] rawNextTimestamp = new byte[32]; + Transaction transaction; + + // Confirm initial value is zero + extractNextTxTimestamp(repository, atAddress, rawNextTimestamp); + assertArrayEquals(new byte[32], rawNextTimestamp); + + // Send message to someone other than AT + sendMessage(repository, deployer, data, deployer.getAddress()); + BlockUtils.mintBlock(repository); + + // Confirm AT does not find message + extractNextTxTimestamp(repository, atAddress, rawNextTimestamp); + assertArrayEquals(new byte[32], rawNextTimestamp); + + // Send message to AT + transaction = sendMessage(repository, deployer, data, atAddress); + BlockUtils.mintBlock(repository); + + // Confirm AT finds message + BlockUtils.mintBlock(repository); + assertTimestamp(repository, atAddress, transaction); + + // Mint a few blocks, then send non-AT message, followed by two AT messages (in same block) + for (int i = 0; i < 5; ++i) + BlockUtils.mintBlock(repository); + + sendMessage(repository, deployer, data, deployer.getAddress()); + + Transaction transaction1 = sendMessage(repository, deployer, data, atAddress); + Transaction transaction2 = sendMessage(repository, deployer, data, atAddress); + + BlockUtils.mintBlock(repository); + + // Confirm AT finds first message + BlockUtils.mintBlock(repository); + assertTimestamp(repository, atAddress, transaction1); + + // Confirm AT finds second message + BlockUtils.mintBlock(repository); + assertTimestamp(repository, atAddress, transaction2); + } + } + + private byte[] buildGetNextTransactionAT() { + // Labels for data segment addresses + int addrCounter = 0; + + // Beginning of data segment for easy extraction + final int addrNextTx = addrCounter; + addrCounter += 4; + + final int addrNextTxIndex = addrCounter++; + + final int addrLastTxTimestamp = addrCounter++; + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // skip addrNextTx + dataByteBuffer.position(dataByteBuffer.position() + 4 * MachineState.VALUE_SIZE); + + // Store pointer to addrNextTx at addrNextTxIndex + dataByteBuffer.putLong(addrNextTx); + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(512); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxTimestamp)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message to AT */ + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxTimestamp)); + // Copy A to data segment, starting at addrNextTx (as pointed to by addrNextTxIndex) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_A_IND, addrNextTxIndex)); + // Stop if timestamp part of A is zero + codeByteBuffer.put(OpCode.STZ_DAT.compile(addrNextTx)); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxTimestamp)); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile AT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "Test AT"; + String description = "Test AT"; + String atType = "Test"; + String tags = "TEST"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private void extractNextTxTimestamp(Repository repository, String atAddress, byte[] rawNextTimestamp) throws DataException { + // Check AT result + ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); + byte[] stateData = atStateData.getStateData(); + + byte[] dataBytes = MachineState.extractDataBytes(stateData); + + System.arraycopy(dataBytes, 0, rawNextTimestamp, 0, rawNextTimestamp.length); + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndImportValid(repository, messageTransactionData, sender); + + return messageTransaction; + } + + private void assertTimestamp(Repository repository, String atAddress, Transaction transaction) throws DataException { + int height = transaction.getHeight(); + byte[] transactionSignature = transaction.getTransactionData().getSignature(); + + BlockData blockData = repository.getBlockRepository().fromHeight(height); + assertNotNull(blockData); + + Block block = new Block(repository, blockData); + + List blockTransactions = block.getTransactions(); + int sequence; + for (sequence = blockTransactions.size() - 1; sequence >= 0; --sequence) + if (Arrays.equals(blockTransactions.get(sequence).getTransactionData().getSignature(), transactionSignature)) + break; + + assertNotSame(-1, sequence); + + byte[] rawNextTimestamp = new byte[32]; + extractNextTxTimestamp(repository, atAddress, rawNextTimestamp); + + Timestamp expectedTimestamp = new Timestamp(height, sequence); + Timestamp actualTimestamp = new Timestamp(BitTwiddling.longFromBEBytes(rawNextTimestamp, 0)); + + assertEquals(String.format("Expected height %d, seq %d but was height %d, seq %d", + height, sequence, + actualTimestamp.blockHeight, actualTimestamp.transactionSequence + ), + expectedTimestamp.longValue(), + actualTimestamp.longValue()); + + byte[] expectedPartialSignature = new byte[24]; + System.arraycopy(transactionSignature, 8, expectedPartialSignature, 0, expectedPartialSignature.length); + + byte[] actualPartialSignature = new byte[24]; + System.arraycopy(rawNextTimestamp, 8, actualPartialSignature, 0, actualPartialSignature.length); + + assertArrayEquals(expectedPartialSignature, actualPartialSignature); + } + +} diff --git a/src/test/java/org/qortal/test/at/GetPartialMessageTests.java b/src/test/java/org/qortal/test/at/GetPartialMessageTests.java new file mode 100644 index 000000000..0f9b188ac --- /dev/null +++ b/src/test/java/org/qortal/test/at/GetPartialMessageTests.java @@ -0,0 +1,219 @@ +package org.qortal.test.at; + +import static org.junit.Assert.*; + +import java.nio.ByteBuffer; + +import org.ciyam.at.CompilationException; +import org.ciyam.at.FunctionCode; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.data.at.ATStateData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; + +public class GetPartialMessageTests extends Common { + + @Before + public void before() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testGetPartialMessage() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + byte[] messageData = "The quick brown fox jumped over the lazy dog.".getBytes(); + int[] offsets = new int[] { 0, 7, 32, 44, messageData.length }; + + byte[] creationBytes = buildGetPartialMessageAT(offsets); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + sendMessage(repository, deployer, messageData, atAddress); + + for (int offset : offsets) { + // Mint another block so AT can process message + BlockUtils.mintBlock(repository); + + byte[] expectedData = new byte[32]; + int byteCount = Math.min(32, messageData.length - offset); + System.arraycopy(messageData, offset, expectedData, 0, byteCount); + + // Check AT result + ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); + byte[] stateData = atStateData.getStateData(); + + byte[] dataBytes = MachineState.extractDataBytes(stateData); + + byte[] actualData = new byte[32]; + System.arraycopy(dataBytes, MachineState.VALUE_SIZE, actualData, 0, 32); + + assertArrayEquals(expectedData, actualData); + } + } + } + + private byte[] buildGetPartialMessageAT(int... offsets) { + // Labels for data segment addresses + int addrCounter = 0; + + final int addrCopyOfBIndex = addrCounter++; + + // 2nd position for easy extraction + final int addrCopyOfB = addrCounter; + addrCounter += 4; + + final int addrResult = addrCounter++; + final int addrLastTxTimestamp = addrCounter++; + final int addrOffset = addrCounter++; + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + dataByteBuffer.putLong(addrCopyOfB); + + // Code labels + Integer labelCheckTx = null; + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(512); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxTimestamp)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message to AT */ + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, OpCode.calcOffset(codeByteBuffer, labelCheckTx))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckTx = codeByteBuffer.position(); + + // Generate code per offset + for (int i = 0; i < offsets.length; ++i) { + if (i > 0) + // Wait for next block + codeByteBuffer.put(OpCode.SLP_IMD.compile()); + + // Set offset + codeByteBuffer.put(OpCode.SET_VAL.compile(addrOffset, offsets[i])); + + // Extract partial message + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrOffset)); + + // Copy B to data segment + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCopyOfBIndex)); + } + + // We're done + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile AT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "Test AT"; + String description = "Test AT"; + String atType = "Test"; + String tags = "TEST"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, messageTransactionData, sender); + + return messageTransaction; + } + +} diff --git a/src/test/java/org/qortal/test/at/SleepUntilMessageOrHeightTests.java b/src/test/java/org/qortal/test/at/SleepUntilMessageOrHeightTests.java new file mode 100644 index 000000000..7ac952d28 --- /dev/null +++ b/src/test/java/org/qortal/test/at/SleepUntilMessageOrHeightTests.java @@ -0,0 +1,365 @@ +package org.qortal.test.at; + +import static org.junit.Assert.*; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import org.ciyam.at.CompilationException; +import org.ciyam.at.FunctionCode; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.ciyam.at.Timestamp; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.Account; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.block.Block; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.utils.BitTwiddling; + +public class SleepUntilMessageOrHeightTests extends Common { + + private static final byte[] messageData = new byte[] { 0x44 }; + private static final byte[] creationBytes = buildSleepUntilMessageOrHeightAT(); + private static final long fundingAmount = 1_00000000L; + private static final long WAKE_HEIGHT = 10L; + + private Repository repository = null; + private PrivateKeyAccount deployer; + private DeployAtTransaction deployAtTransaction; + private Account atAccount; + private String atAddress; + private byte[] rawNextTimestamp = new byte[32]; + private Transaction transaction; + + @Before + public void before() throws DataException { + Common.useDefaultSettings(); + + this.repository = RepositoryManager.getRepository(); + this.deployer = Common.getTestAccount(repository, "alice"); + + this.deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + this.atAccount = deployAtTransaction.getATAccount(); + this.atAddress = deployAtTransaction.getATAccount().getAddress(); + } + + @After + public void after() throws DataException { + if (this.repository != null) + this.repository.close(); + + this.repository = null; + } + + @Test + public void testDeploy() throws DataException { + // Confirm initial value is zero + extractNextTxTimestamp(repository, atAddress, rawNextTimestamp); + assertArrayEquals(new byte[32], rawNextTimestamp); + } + + @Test + public void testFeelessSleep() throws DataException { + // Mint block to allow AT to initialize and call SLEEP_UNTIL_MESSAGE + BlockUtils.mintBlock(repository); + + // Fetch AT's balance for this height + long preMintBalance = atAccount.getConfirmedBalance(Asset.QORT); + + // Mint block + BlockUtils.mintBlock(repository); + + // Fetch new AT balance + long postMintBalance = atAccount.getConfirmedBalance(Asset.QORT); + + assertEquals(preMintBalance, postMintBalance); + } + + @Test + public void testFeelessSleep2() throws DataException { + // Mint block to allow AT to initialize and call SLEEP_UNTIL_MESSAGE + BlockUtils.mintBlock(repository); + + // Fetch AT's balance for this height + long preMintBalance = atAccount.getConfirmedBalance(Asset.QORT); + + // Mint several blocks + for (int i = 0; i < 5; ++i) + BlockUtils.mintBlock(repository); + + // Fetch new AT balance + long postMintBalance = atAccount.getConfirmedBalance(Asset.QORT); + + assertEquals(preMintBalance, postMintBalance); + } + + @Test + public void testSleepUntilMessage() throws DataException { + // Mint block to allow AT to initialize and call SLEEP_UNTIL_MESSAGE_OR_HEIGHT + BlockUtils.mintBlock(repository); + + // Send message to AT + transaction = sendMessage(repository, deployer, messageData, atAddress); + BlockUtils.mintBlock(repository); + + // Mint block so AT executes and finds message + BlockUtils.mintBlock(repository); + + // Confirm AT finds message + assertTimestamp(repository, atAddress, transaction); + } + + @Test + public void testSleepUntilHeight() throws DataException { + // AT deployment in block 2 + + // Mint block to allow AT to initialize and call SLEEP_UNTIL_MESSAGE_OR_HEIGHT + BlockUtils.mintBlock(repository); // height now 3 + + // Fetch AT's balance for this height + long preMintBalance = atAccount.getConfirmedBalance(Asset.QORT); + + // Mint several blocks + for (int i = 3; i < WAKE_HEIGHT; ++i) + BlockUtils.mintBlock(repository); + + // We should now be at WAKE_HEIGHT + long height = repository.getBlockRepository().getBlockchainHeight(); + assertEquals(WAKE_HEIGHT, height); + + // AT should have woken and run at this height so balance should have changed + + // Fetch new AT balance + long postMintBalance = atAccount.getConfirmedBalance(Asset.QORT); + + assertNotSame(preMintBalance, postMintBalance); + + // Confirm AT has no message + extractNextTxTimestamp(repository, atAddress, rawNextTimestamp); + assertArrayEquals(new byte[32], rawNextTimestamp); + + // Mint yet another block + BlockUtils.mintBlock(repository); + + // AT should also have woken and run at this height so balance should have changed + + // Fetch new AT balance + long postMint2Balance = atAccount.getConfirmedBalance(Asset.QORT); + + assertNotSame(postMintBalance, postMint2Balance); + + // Confirm AT still has no message + extractNextTxTimestamp(repository, atAddress, rawNextTimestamp); + assertArrayEquals(new byte[32], rawNextTimestamp); + + } + + private static byte[] buildSleepUntilMessageOrHeightAT() { + // Labels for data segment addresses + int addrCounter = 0; + + // Beginning of data segment for easy extraction + final int addrNextTx = addrCounter; + addrCounter += 4; + + final int addrNextTxIndex = addrCounter++; + + final int addrLastTxTimestamp = addrCounter++; + + final int addrWakeHeight = addrCounter++; + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // skip addrNextTx + dataByteBuffer.position(dataByteBuffer.position() + 4 * MachineState.VALUE_SIZE); + + // Store pointer to addrNextTx at addrNextTxIndex + dataByteBuffer.putLong(addrNextTx); + + // skip addrLastTxTimestamp + dataByteBuffer.position(dataByteBuffer.position() + MachineState.VALUE_SIZE); + + // Store fixed wake height (block 10) + dataByteBuffer.putLong(WAKE_HEIGHT); + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(512); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxTimestamp)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message to AT */ + + /* Sleep until message arrives */ + codeByteBuffer.put(OpCode.EXT_FUN_DAT_2.compile(QortalFunctionCode.SLEEP_UNTIL_MESSAGE_OR_HEIGHT.value, addrLastTxTimestamp, addrWakeHeight)); + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxTimestamp)); + + // Copy A to data segment, starting at addrNextTx (as pointed to by addrNextTxIndex) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_A_IND, addrNextTxIndex)); + + // Stop if timestamp part of A is zero + codeByteBuffer.put(OpCode.STZ_DAT.compile(addrNextTx)); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxTimestamp)); + + // We're done + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile AT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "Test AT"; + String description = "Test AT"; + String atType = "Test"; + String tags = "TEST"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private void extractNextTxTimestamp(Repository repository, String atAddress, byte[] rawNextTimestamp) throws DataException { + // Check AT result + ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); + byte[] stateData = atStateData.getStateData(); + + byte[] dataBytes = MachineState.extractDataBytes(stateData); + + System.arraycopy(dataBytes, 0, rawNextTimestamp, 0, rawNextTimestamp.length); + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndImportValid(repository, messageTransactionData, sender); + + return messageTransaction; + } + + private void assertTimestamp(Repository repository, String atAddress, Transaction transaction) throws DataException { + int height = transaction.getHeight(); + byte[] transactionSignature = transaction.getTransactionData().getSignature(); + + BlockData blockData = repository.getBlockRepository().fromHeight(height); + assertNotNull(blockData); + + Block block = new Block(repository, blockData); + + List blockTransactions = block.getTransactions(); + int sequence; + for (sequence = blockTransactions.size() - 1; sequence >= 0; --sequence) + if (Arrays.equals(blockTransactions.get(sequence).getTransactionData().getSignature(), transactionSignature)) + break; + + assertNotSame(-1, sequence); + + byte[] rawNextTimestamp = new byte[32]; + extractNextTxTimestamp(repository, atAddress, rawNextTimestamp); + + Timestamp expectedTimestamp = new Timestamp(height, sequence); + Timestamp actualTimestamp = new Timestamp(BitTwiddling.longFromBEBytes(rawNextTimestamp, 0)); + + assertEquals(String.format("Expected height %d, seq %d but was height %d, seq %d", + height, sequence, + actualTimestamp.blockHeight, actualTimestamp.transactionSequence + ), + expectedTimestamp.longValue(), + actualTimestamp.longValue()); + + byte[] expectedPartialSignature = new byte[24]; + System.arraycopy(transactionSignature, 8, expectedPartialSignature, 0, expectedPartialSignature.length); + + byte[] actualPartialSignature = new byte[24]; + System.arraycopy(rawNextTimestamp, 8, actualPartialSignature, 0, actualPartialSignature.length); + + assertArrayEquals(expectedPartialSignature, actualPartialSignature); + } + +} diff --git a/src/test/java/org/qortal/test/at/SleepUntilMessageTests.java b/src/test/java/org/qortal/test/at/SleepUntilMessageTests.java new file mode 100644 index 000000000..290f973a9 --- /dev/null +++ b/src/test/java/org/qortal/test/at/SleepUntilMessageTests.java @@ -0,0 +1,311 @@ +package org.qortal.test.at; + +import static org.junit.Assert.*; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import org.ciyam.at.CompilationException; +import org.ciyam.at.FunctionCode; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.ciyam.at.Timestamp; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.Account; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.block.Block; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.utils.BitTwiddling; + +public class SleepUntilMessageTests extends Common { + + private static final byte[] messageData = new byte[] { 0x44 }; + private static final byte[] creationBytes = buildSleepUntilMessageAT(); + private static final long fundingAmount = 1_00000000L; + + private Repository repository = null; + private PrivateKeyAccount deployer; + private DeployAtTransaction deployAtTransaction; + private Account atAccount; + private String atAddress; + private byte[] rawNextTimestamp = new byte[32]; + private Transaction transaction; + + @Before + public void before() throws DataException { + Common.useDefaultSettings(); + + this.repository = RepositoryManager.getRepository(); + this.deployer = Common.getTestAccount(repository, "alice"); + + this.deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + this.atAccount = deployAtTransaction.getATAccount(); + this.atAddress = deployAtTransaction.getATAccount().getAddress(); + } + + @After + public void after() throws DataException { + if (this.repository != null) + this.repository.close(); + + this.repository = null; + } + + @Test + public void testDeploy() throws DataException { + // Confirm initial value is zero + extractNextTxTimestamp(repository, atAddress, rawNextTimestamp); + assertArrayEquals(new byte[32], rawNextTimestamp); + } + + @Test + public void testFeelessSleep() throws DataException { + // Mint block to allow AT to initialize and call SLEEP_UNTIL_MESSAGE + BlockUtils.mintBlock(repository); + + // Fetch AT's balance for this height + long preMintBalance = atAccount.getConfirmedBalance(Asset.QORT); + + // Mint block + BlockUtils.mintBlock(repository); + + // Fetch new AT balance + long postMintBalance = atAccount.getConfirmedBalance(Asset.QORT); + + assertEquals(preMintBalance, postMintBalance); + } + + @Test + public void testFeelessSleep2() throws DataException { + // Mint block to allow AT to initialize and call SLEEP_UNTIL_MESSAGE + BlockUtils.mintBlock(repository); + + // Fetch AT's balance for this height + long preMintBalance = atAccount.getConfirmedBalance(Asset.QORT); + + // Mint several blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + + // Fetch new AT balance + long postMintBalance = atAccount.getConfirmedBalance(Asset.QORT); + + assertEquals(preMintBalance, postMintBalance); + } + + @Test + public void testSleepUntilMessage() throws DataException { + // Mint block to allow AT to initialize and call SLEEP_UNTIL_MESSAGE + BlockUtils.mintBlock(repository); + + // Send message to AT + transaction = sendMessage(repository, deployer, messageData, atAddress); + BlockUtils.mintBlock(repository); + + // Mint block so AT executes and finds message + BlockUtils.mintBlock(repository); + + // Confirm AT finds message + assertTimestamp(repository, atAddress, transaction); + } + + private static byte[] buildSleepUntilMessageAT() { + // Labels for data segment addresses + int addrCounter = 0; + + // Beginning of data segment for easy extraction + final int addrNextTx = addrCounter; + addrCounter += 4; + + final int addrNextTxIndex = addrCounter++; + + final int addrLastTxTimestamp = addrCounter++; + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // skip addrNextTx + dataByteBuffer.position(dataByteBuffer.position() + 4 * MachineState.VALUE_SIZE); + + // Store pointer to addrNextTx at addrNextTxIndex + dataByteBuffer.putLong(addrNextTx); + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(512); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxTimestamp)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message to AT */ + + /* Sleep until message arrives */ + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.SLEEP_UNTIL_MESSAGE.value, addrLastTxTimestamp)); + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxTimestamp)); + + // Copy A to data segment, starting at addrNextTx (as pointed to by addrNextTxIndex) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_A_IND, addrNextTxIndex)); + + // Stop if timestamp part of A is zero + codeByteBuffer.put(OpCode.STZ_DAT.compile(addrNextTx)); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxTimestamp)); + + // We're done + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile AT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "Test AT"; + String description = "Test AT"; + String atType = "Test"; + String tags = "TEST"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private void extractNextTxTimestamp(Repository repository, String atAddress, byte[] rawNextTimestamp) throws DataException { + // Check AT result + ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); + byte[] stateData = atStateData.getStateData(); + + byte[] dataBytes = MachineState.extractDataBytes(stateData); + + System.arraycopy(dataBytes, 0, rawNextTimestamp, 0, rawNextTimestamp.length); + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndImportValid(repository, messageTransactionData, sender); + + return messageTransaction; + } + + private void assertTimestamp(Repository repository, String atAddress, Transaction transaction) throws DataException { + int height = transaction.getHeight(); + byte[] transactionSignature = transaction.getTransactionData().getSignature(); + + BlockData blockData = repository.getBlockRepository().fromHeight(height); + assertNotNull(blockData); + + Block block = new Block(repository, blockData); + + List blockTransactions = block.getTransactions(); + int sequence; + for (sequence = blockTransactions.size() - 1; sequence >= 0; --sequence) + if (Arrays.equals(blockTransactions.get(sequence).getTransactionData().getSignature(), transactionSignature)) + break; + + assertNotSame(-1, sequence); + + byte[] rawNextTimestamp = new byte[32]; + extractNextTxTimestamp(repository, atAddress, rawNextTimestamp); + + Timestamp expectedTimestamp = new Timestamp(height, sequence); + Timestamp actualTimestamp = new Timestamp(BitTwiddling.longFromBEBytes(rawNextTimestamp, 0)); + + assertEquals(String.format("Expected height %d, seq %d but was height %d, seq %d", + height, sequence, + actualTimestamp.blockHeight, actualTimestamp.transactionSequence + ), + expectedTimestamp.longValue(), + actualTimestamp.longValue()); + + byte[] expectedPartialSignature = new byte[24]; + System.arraycopy(transactionSignature, 8, expectedPartialSignature, 0, expectedPartialSignature.length); + + byte[] actualPartialSignature = new byte[24]; + System.arraycopy(rawNextTimestamp, 8, actualPartialSignature, 0, actualPartialSignature.length); + + assertArrayEquals(expectedPartialSignature, actualPartialSignature); + } + +} diff --git a/src/test/java/org/qortal/test/btcacct/AtTests.java b/src/test/java/org/qortal/test/btcacct/AtTests.java deleted file mode 100644 index 19fd73401..000000000 --- a/src/test/java/org/qortal/test/btcacct/AtTests.java +++ /dev/null @@ -1,555 +0,0 @@ -package org.qortal.test.btcacct; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.time.format.FormatStyle; -import java.util.Arrays; -import java.util.List; -import java.util.function.Function; - -import org.bitcoinj.core.Base58; -import org.junit.Before; -import org.junit.Test; -import org.qortal.account.Account; -import org.qortal.account.PrivateKeyAccount; -import org.qortal.asset.Asset; -import org.qortal.crosschain.BTCACCT; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.at.ATStateData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.test.common.BlockUtils; -import org.qortal.test.common.Common; -import org.qortal.test.common.TransactionUtils; -import org.qortal.transaction.DeployAtTransaction; -import org.qortal.transaction.MessageTransaction; -import org.qortal.utils.Amounts; - -import com.google.common.hash.HashCode; -import com.google.common.primitives.Bytes; - -public class AtTests extends Common { - - public static final byte[] secret = "This string is exactly 32 bytes!".getBytes(); - public static final byte[] secretHash = Crypto.hash160(secret); // daf59884b4d1aec8c1b17102530909ee43c0151a - public static final int refundTimeout = 10; // blocks - public static final long initialPayout = 100000L; - public static final long redeemAmount = 80_40200000L; - public static final long fundingAmount = 123_45600000L; - public static final long bitcoinAmount = 864200L; - - @Before - public void beforeTest() throws DataException { - Common.useDefaultSettings(); - } - - @Test - public void testCompile() { - Account deployer = Common.getTestAccount(null, "chloe"); - - byte[] creationBytes = BTCACCT.buildQortalAT(deployer.getAddress(), secretHash, refundTimeout, initialPayout, redeemAmount, bitcoinAmount); - System.out.println("CIYAM AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); - } - - @Test - public void testDeploy() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount recipient = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long recipientsInitialBalance = recipient.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer); - - long expectedBalance = deployersInitialBalance - fundingAmount - deployAtTransaction.getTransactionData().getFee(); - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-deployment balance incorrect", expectedBalance, actualBalance); - - expectedBalance = fundingAmount; - actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); - - assertEquals("AT's post-deployment balance incorrect", expectedBalance, actualBalance); - - expectedBalance = recipientsInitialBalance; - actualBalance = recipient.getConfirmedBalance(Asset.QORT); - - assertEquals("Recipient's post-deployment balance incorrect", expectedBalance, actualBalance); - - // Test orphaning - BlockUtils.orphanLastBlock(repository); - - expectedBalance = deployersInitialBalance; - actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); - - expectedBalance = 0; - actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); - - assertEquals("AT's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); - - expectedBalance = recipientsInitialBalance; - actualBalance = recipient.getConfirmedBalance(Asset.QORT); - - assertEquals("Recipient's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); - } - } - - @SuppressWarnings("unused") - @Test - public void testOfferCancel() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount recipient = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long recipientsInitialBalance = recipient.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; - - // Send creator's address to AT - byte[] recipientAddressBytes = Bytes.ensureCapacity(Base58.decode(deployer.getAddress()), 32, 0); - MessageTransaction messageTransaction = sendMessage(repository, deployer, recipientAddressBytes, atAddress); - long messageFee = messageTransaction.getTransactionData().getFee(); - - // Refund should happen 1st block after receiving recipient address - BlockUtils.mintBlock(repository); - - long expectedMinimumBalance = deployersPostDeploymentBalance; - long expectedMaximumBalance = deployersInitialBalance - deployAtFee - messageFee; - - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); - assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); - - describeAt(repository, atAddress); - - // Test orphaning - BlockUtils.orphanLastBlock(repository); - - long expectedBalance = deployersPostDeploymentBalance - messageFee; - actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); - } - } - - @SuppressWarnings("unused") - @Test - public void testInitialPayment() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount recipient = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long recipientsInitialBalance = recipient.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - // Send recipient's address to AT - byte[] recipientAddressBytes = Bytes.ensureCapacity(Base58.decode(recipient.getAddress()), 32, 0); - MessageTransaction messageTransaction = sendMessage(repository, deployer, recipientAddressBytes, atAddress); - - // Initial payment should happen 1st block after receiving recipient address - BlockUtils.mintBlock(repository); - - long expectedBalance = recipientsInitialBalance + initialPayout; - long actualBalance = recipient.getConfirmedBalance(Asset.QORT); - - assertEquals("Recipient's post-initial-payout balance incorrect", expectedBalance, actualBalance); - - describeAt(repository, atAddress); - - // Test orphaning - BlockUtils.orphanLastBlock(repository); - - expectedBalance = recipientsInitialBalance; - actualBalance = recipient.getConfirmedBalance(Asset.QORT); - - assertEquals("Recipient's pre-initial-payout balance incorrect", expectedBalance, actualBalance); - } - } - - // TEST SENDING RECIPIENT ADDRESS BUT NOT FROM AT CREATOR (SHOULD BE IGNORED) - @SuppressWarnings("unused") - @Test - public void testIncorrectTradeSender() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount recipient = Common.getTestAccount(repository, "dilbert"); - PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long recipientsInitialBalance = recipient.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - // Send recipient's address to AT BUT NOT FROM AT CREATOR - byte[] recipientAddressBytes = Bytes.ensureCapacity(Base58.decode(recipient.getAddress()), 32, 0); - MessageTransaction messageTransaction = sendMessage(repository, bystander, recipientAddressBytes, atAddress); - - // Initial payment should NOT happen - BlockUtils.mintBlock(repository); - - long expectedBalance = recipientsInitialBalance; - long actualBalance = recipient.getConfirmedBalance(Asset.QORT); - - assertEquals("Recipient's post-initial-payout balance incorrect", expectedBalance, actualBalance); - - describeAt(repository, atAddress); - } - } - - @SuppressWarnings("unused") - @Test - public void testAutomaticTradeRefund() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount recipient = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long recipientsInitialBalance = recipient.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - // Send recipient's address to AT - byte[] recipientAddressBytes = Bytes.ensureCapacity(Base58.decode(recipient.getAddress()), 32, 0); - MessageTransaction messageTransaction = sendMessage(repository, deployer, recipientAddressBytes, atAddress); - - // Initial payment should happen 1st block after receiving recipient address - BlockUtils.mintBlock(repository); - - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - long messageFee = messageTransaction.getTransactionData().getFee(); - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee - messageFee; - - checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); - - describeAt(repository, atAddress); - - // Test orphaning - BlockUtils.orphanLastBlock(repository); - BlockUtils.orphanLastBlock(repository); - - long expectedBalance = deployersPostDeploymentBalance; - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); - } - } - - @SuppressWarnings("unused") - @Test - public void testCorrectSecretCorrectSender() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount recipient = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long recipientsInitialBalance = recipient.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - // Send recipient's address to AT - byte[] recipientAddressBytes = Bytes.ensureCapacity(Base58.decode(recipient.getAddress()), 32, 0); - MessageTransaction messageTransaction = sendMessage(repository, deployer, recipientAddressBytes, atAddress); - - // Initial payment should happen 1st block after receiving recipient address - BlockUtils.mintBlock(repository); - - // Send correct secret to AT - messageTransaction = sendMessage(repository, recipient, secret, atAddress); - - // AT should send funds in the next block - ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); - BlockUtils.mintBlock(repository); - - long expectedBalance = recipientsInitialBalance + initialPayout - messageTransaction.getTransactionData().getFee() + redeemAmount; - long actualBalance = recipient.getConfirmedBalance(Asset.QORT); - - assertEquals("Recipent's post-redeem balance incorrect", expectedBalance, actualBalance); - - describeAt(repository, atAddress); - - // Orphan redeem - BlockUtils.orphanLastBlock(repository); - - expectedBalance = recipientsInitialBalance + initialPayout - messageTransaction.getTransactionData().getFee(); - actualBalance = recipient.getConfirmedBalance(Asset.QORT); - - assertEquals("Recipent's post-orphan/pre-redeem balance incorrect", expectedBalance, actualBalance); - - // Check AT state - ATStateData postOrphanAtStateData = repository.getATRepository().getLatestATState(atAddress); - - assertTrue("AT states mismatch", Arrays.equals(preRedeemAtStateData.getStateData(), postOrphanAtStateData.getStateData())); - } - } - - @SuppressWarnings("unused") - @Test - public void testCorrectSecretIncorrectSender() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount recipient = Common.getTestAccount(repository, "dilbert"); - PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long recipientsInitialBalance = recipient.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer); - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - // Send recipient's address to AT - byte[] recipientAddressBytes = Bytes.ensureCapacity(Base58.decode(recipient.getAddress()), 32, 0); - MessageTransaction messageTransaction = sendMessage(repository, deployer, recipientAddressBytes, atAddress); - - // Initial payment should happen 1st block after receiving recipient address - BlockUtils.mintBlock(repository); - - // Send correct secret to AT, but from wrong account - messageTransaction = sendMessage(repository, bystander, secret, atAddress); - - // AT should NOT send funds in the next block - ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); - BlockUtils.mintBlock(repository); - - long expectedBalance = recipientsInitialBalance + initialPayout; - long actualBalance = recipient.getConfirmedBalance(Asset.QORT); - - assertEquals("Recipent's balance incorrect", expectedBalance, actualBalance); - - describeAt(repository, atAddress); - - checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); - } - } - - @SuppressWarnings("unused") - @Test - public void testIncorrectSecretCorrectSender() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount recipient = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long recipientsInitialBalance = recipient.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer); - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - // Send recipient's address to AT - byte[] recipientAddressBytes = Bytes.ensureCapacity(Base58.decode(recipient.getAddress()), 32, 0); - MessageTransaction messageTransaction = sendMessage(repository, deployer, recipientAddressBytes, atAddress); - - // Initial payment should happen 1st block after receiving recipient address - BlockUtils.mintBlock(repository); - - // Send correct secret to AT, but from wrong account - byte[] wrongSecret = Crypto.digest(secret); - messageTransaction = sendMessage(repository, recipient, wrongSecret, atAddress); - - // AT should NOT send funds in the next block - ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); - BlockUtils.mintBlock(repository); - - long expectedBalance = recipientsInitialBalance + initialPayout - messageTransaction.getTransactionData().getFee(); - long actualBalance = recipient.getConfirmedBalance(Asset.QORT); - - assertEquals("Recipent's balance incorrect", expectedBalance, actualBalance); - - describeAt(repository, atAddress); - - checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); - } - } - - @SuppressWarnings("unused") - @Test - public void testDescribeDeployed() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount recipient = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long recipientsInitialBalance = recipient.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer); - - List executableAts = repository.getATRepository().getAllExecutableATs(); - - for (ATData atData : executableAts) { - String atAddress = atData.getATAddress(); - byte[] codeBytes = atData.getCodeBytes(); - byte[] codeHash = Crypto.digest(codeBytes); - - System.out.println(String.format("%s: code length: %d byte%s, code hash: %s", - atAddress, - codeBytes.length, - (codeBytes.length != 1 ? "s": ""), - HashCode.fromBytes(codeHash))); - - // Not one of ours? - if (!Arrays.equals(codeHash, BTCACCT.CODE_BYTES_HASH)) - continue; - - describeAt(repository, atAddress); - } - } - } - - private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer) throws DataException { - byte[] creationBytes = BTCACCT.buildQortalAT(deployer.getAddress(), secretHash, refundTimeout, initialPayout, redeemAmount, bitcoinAmount); - - long txTimestamp = System.currentTimeMillis(); - byte[] lastReference = deployer.getLastReference(); - - if (lastReference == null) { - System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); - System.exit(2); - } - - Long fee = null; - String name = "QORT-BTC cross-chain trade"; - String description = String.format("Qortal-Bitcoin cross-chain trade"); - String atType = "ACCT"; - String tags = "QORT-BTC ACCT"; - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); - TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); - - DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); - - return deployAtTransaction; - } - - private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { - long txTimestamp = System.currentTimeMillis(); - byte[] lastReference = sender.getLastReference(); - - if (lastReference == null) { - System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); - System.exit(2); - } - - Long fee = null; - int version = 4; - int nonce = 0; - long amount = 0; - Long assetId = null; // because amount is zero - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); - TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); - - MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); - - fee = messageTransaction.calcRecommendedFee(); - messageTransactionData.setFee(fee); - - TransactionUtils.signAndMint(repository, messageTransactionData, sender); - - return messageTransaction; - } - - private void checkTradeRefund(Repository repository, Account deployer, long deployersInitialBalance, long deployAtFee) throws DataException { - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; - - // AT should automatically refund deployer after 'refundTimeout' blocks - for (int blockCount = 0; blockCount <= refundTimeout; ++blockCount) - BlockUtils.mintBlock(repository); - - // We don't bother to exactly calculate QORT spent running AT for several blocks, but we do know the expected range - long expectedMinimumBalance = deployersPostDeploymentBalance; - long expectedMaximumBalance = deployersInitialBalance - deployAtFee - initialPayout; - - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); - assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); - } - - private void describeAt(Repository repository, String atAddress) throws DataException { - ATData atData = repository.getATRepository().fromATAddress(atAddress); - CrossChainTradeData tradeData = BTCACCT.populateTradeData(repository, atData); - - Function epochMilliFormatter = (timestamp) -> LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneOffset.UTC).format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM)); - int currentBlockHeight = repository.getBlockRepository().getBlockchainHeight(); - - System.out.print(String.format("%s:\n" - + "\tcreator: %s,\n" - + "\tcreation timestamp: %s,\n" - + "\tcurrent balance: %s QORT,\n" - + "\tHASH160 of secret: %s,\n" - + "\tinitial payout: %s QORT,\n" - + "\tredeem payout: %s QORT,\n" - + "\texpected bitcoin: %s BTC,\n" - + "\ttrade timeout: %d minutes (from trade start),\n" - + "\tcurrent block height: %d,\n", - tradeData.qortalAtAddress, - tradeData.qortalCreator, - epochMilliFormatter.apply(tradeData.creationTimestamp), - Amounts.prettyAmount(tradeData.qortBalance), - HashCode.fromBytes(tradeData.secretHash).toString().substring(0, 40), - Amounts.prettyAmount(tradeData.initialPayout), - Amounts.prettyAmount(tradeData.redeemPayout), - Amounts.prettyAmount(tradeData.expectedBitcoin), - tradeData.tradeRefundTimeout, - currentBlockHeight)); - - // Are we in 'offer' or 'trade' stage? - if (tradeData.tradeRefundHeight == null) { - // Offer - System.out.println(String.format("\tstatus: 'offer mode'")); - } else { - // Trade - System.out.println(String.format("\tstatus: 'trade mode',\n" - + "\ttrade timeout: block %d,\n" - + "\tBitcoin P2SH nLockTime: %d (%s),\n" - + "\ttrade recipient: %s", - tradeData.tradeRefundHeight, - tradeData.lockTime, epochMilliFormatter.apply(tradeData.lockTime * 1000L), - tradeData.qortalRecipient)); - } - } - -} diff --git a/src/test/java/org/qortal/test/btcacct/BtcTests.java b/src/test/java/org/qortal/test/btcacct/BtcTests.java deleted file mode 100644 index d0530c476..000000000 --- a/src/test/java/org/qortal/test/btcacct/BtcTests.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.qortal.test.btcacct; - -import static org.junit.Assert.*; - -import java.util.Arrays; -import java.util.List; - -import org.bitcoinj.store.BlockStoreException; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.qortal.crosschain.BTC; -import org.qortal.crosschain.BTCACCT; -import org.qortal.repository.DataException; -import org.qortal.test.common.Common; - -public class BtcTests extends Common { - - @Before - public void beforeTest() throws DataException { - Common.useDefaultSettings(); // TestNet3 - } - - @After - public void afterTest() { - BTC.resetForTesting(); - } - - @Test - public void testGetMedianBlockTime() throws BlockStoreException { - System.out.println(String.format("Starting BTC instance...")); - BTC btc = BTC.getInstance(); - System.out.println(String.format("BTC instance started")); - - long before = System.currentTimeMillis(); - System.out.println(String.format("Bitcoin median blocktime: %d", btc.getMedianBlockTime())); - long afterFirst = System.currentTimeMillis(); - - System.out.println(String.format("Bitcoin median blocktime: %d", btc.getMedianBlockTime())); - long afterSecond = System.currentTimeMillis(); - - long firstPeriod = afterFirst - before; - long secondPeriod = afterSecond - afterFirst; - - System.out.println(String.format("1st call: %d ms, 2nd call: %d ms", firstPeriod, secondPeriod)); - - assertTrue("2nd call should be quicker than 1st", secondPeriod < firstPeriod); - assertTrue("2nd call should take less than 5 seconds", secondPeriod < 5000L); - } - - @Test - public void testFindP2shSecret() { - // This actually exists on TEST3 but can take a while to fetch - String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; - - List rawTransactions = BTC.getInstance().getAddressTransactions(p2shAddress); - - byte[] expectedSecret = AtTests.secret; - byte[] secret = BTCACCT.findP2shSecret(p2shAddress, rawTransactions); - - assertNotNull(secret); - assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret)); - } - -} diff --git a/src/test/java/org/qortal/test/btcacct/BuildP2SH.java b/src/test/java/org/qortal/test/btcacct/BuildP2SH.java deleted file mode 100644 index 25a95d8b1..000000000 --- a/src/test/java/org/qortal/test/btcacct/BuildP2SH.java +++ /dev/null @@ -1,126 +0,0 @@ -package org.qortal.test.btcacct; - -import java.security.Security; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.script.Script.ScriptType; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.qortal.controller.Controller; -import org.qortal.crosschain.BTC; -import org.qortal.crosschain.BTCACCT; -import org.qortal.crypto.Crypto; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryFactory; -import org.qortal.repository.RepositoryManager; -import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; -import org.qortal.settings.Settings; - -import com.google.common.hash.HashCode; - -public class BuildP2SH { - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: BuildP2SH ()")); - System.err.println(String.format("example: BuildP2SH " - + "mrTDPdM15cFWJC4g223BXX5snicfVJBx6M \\\n" - + "\t0.00008642 \\\n" - + "\tn2N5VKrzq39nmuefZwp3wBiF4icdXX2B6o \\\n" - + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" - + "\t1585920000")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length < 5 || args.length > 6) - usage(null); - - Security.insertProviderAt(new BouncyCastleProvider(), 0); - Settings.fileInstance("settings-test.json"); - - BTC btc = BTC.getInstance(); - NetworkParameters params = btc.getNetworkParameters(); - - Address refundBitcoinAddress = null; - Coin bitcoinAmount = null; - Address redeemBitcoinAddress = null; - byte[] secretHash = null; - int lockTime = 0; - Coin bitcoinFee = Common.DEFAULT_BTC_FEE; - - int argIndex = 0; - try { - refundBitcoinAddress = Address.fromString(params, args[argIndex++]); - if (refundBitcoinAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Refund BTC address must be in P2PKH form"); - - bitcoinAmount = Coin.parseCoin(args[argIndex++]); - - redeemBitcoinAddress = Address.fromString(params, args[argIndex++]); - if (redeemBitcoinAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Redeem BTC address must be in P2PKH form"); - - secretHash = HashCode.fromString(args[argIndex++]).asBytes(); - if (secretHash.length != 20) - usage("Hash of secret must be 20 bytes"); - - lockTime = Integer.parseInt(args[argIndex++]); - int refundTimeoutDelay = lockTime - (int) (System.currentTimeMillis() / 1000L); - if (refundTimeoutDelay < 600 || refundTimeoutDelay > 30 * 24 * 60 * 60) - usage("Locktime (seconds) should be at between 10 minutes and 1 month from now"); - - if (args.length > argIndex) - bitcoinFee = Coin.parseCoin(args[argIndex++]); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - try { - RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); - RepositoryManager.setRepositoryFactory(repositoryFactory); - } catch (DataException e) { - throw new RuntimeException("Repository startup issue: " + e.getMessage()); - } - - try (final Repository repository = RepositoryManager.getRepository()) { - System.out.println("Confirm the following is correct based on the info you've given:"); - - System.out.println(String.format("Refund Bitcoin address: %s", refundBitcoinAddress)); - System.out.println(String.format("Bitcoin redeem amount: %s", bitcoinAmount.toPlainString())); - - System.out.println(String.format("Redeem Bitcoin address: %s", redeemBitcoinAddress)); - System.out.println(String.format("Redeem miner's fee: %s", BTC.FORMAT.format(bitcoinFee))); - - System.out.println(String.format("Redeem script lockTime: %s (%d)", LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC), lockTime)); - System.out.println(String.format("Hash of secret: %s", HashCode.fromBytes(secretHash))); - - byte[] redeemScriptBytes = BTCACCT.buildScript(refundBitcoinAddress.getHash(), lockTime, redeemBitcoinAddress.getHash(), secretHash); - System.out.println(String.format("Redeem script: %s", HashCode.fromBytes(redeemScriptBytes))); - - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - - Address p2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - System.out.println(String.format("P2SH address: %s", p2shAddress)); - - bitcoinAmount = bitcoinAmount.add(bitcoinFee); - - // Fund P2SH - System.out.println(String.format("\nYou need to fund %s with %s (includes redeem/refund fee of %s)", - p2shAddress.toString(), BTC.FORMAT.format(bitcoinAmount), BTC.FORMAT.format(bitcoinFee))); - - System.out.println("Once this is done, responder should run Respond to check P2SH funding and create AT"); - } catch (DataException e) { - throw new RuntimeException("Repository issue: " + e.getMessage()); - } - } - -} diff --git a/src/test/java/org/qortal/test/btcacct/CheckP2SH.java b/src/test/java/org/qortal/test/btcacct/CheckP2SH.java deleted file mode 100644 index 8313d573b..000000000 --- a/src/test/java/org/qortal/test/btcacct/CheckP2SH.java +++ /dev/null @@ -1,171 +0,0 @@ -package org.qortal.test.btcacct; - -import java.security.Security; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.List; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.core.TransactionOutput; -import org.bitcoinj.script.Script.ScriptType; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.qortal.controller.Controller; -import org.qortal.crosschain.BTC; -import org.qortal.crosschain.BTCACCT; -import org.qortal.crypto.Crypto; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryFactory; -import org.qortal.repository.RepositoryManager; -import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; -import org.qortal.settings.Settings; - -import com.google.common.hash.HashCode; - -public class CheckP2SH { - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: CheckP2SH ()")); - System.err.println(String.format("example: CheckP2SH " - + "2NEZboTLhBDPPQciR7sExBhy3TsDi7wV3Cv \\\n" - + "mrTDPdM15cFWJC4g223BXX5snicfVJBx6M \\\n" - + "\t0.00008642 \\\n" - + "\tn2N5VKrzq39nmuefZwp3wBiF4icdXX2B6o \\\n" - + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" - + "\t1585920000")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length < 6 || args.length > 7) - usage(null); - - Security.insertProviderAt(new BouncyCastleProvider(), 0); - Settings.fileInstance("settings-test.json"); - - BTC btc = BTC.getInstance(); - NetworkParameters params = btc.getNetworkParameters(); - - Address p2shAddress = null; - Address refundBitcoinAddress = null; - Coin bitcoinAmount = null; - Address redeemBitcoinAddress = null; - byte[] secretHash = null; - int lockTime = 0; - Coin bitcoinFee = Common.DEFAULT_BTC_FEE; - - int argIndex = 0; - try { - p2shAddress = Address.fromString(params, args[argIndex++]); - if (p2shAddress.getOutputScriptType() != ScriptType.P2SH) - usage("P2SH address invalid"); - - refundBitcoinAddress = Address.fromString(params, args[argIndex++]); - if (refundBitcoinAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Refund BTC address must be in P2PKH form"); - - bitcoinAmount = Coin.parseCoin(args[argIndex++]); - - redeemBitcoinAddress = Address.fromString(params, args[argIndex++]); - if (redeemBitcoinAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Redeem BTC address must be in P2PKH form"); - - secretHash = HashCode.fromString(args[argIndex++]).asBytes(); - if (secretHash.length != 20) - usage("Hash of secret must be 20 bytes"); - - lockTime = Integer.parseInt(args[argIndex++]); - int refundTimeoutDelay = lockTime - (int) (System.currentTimeMillis() / 1000L); - if (refundTimeoutDelay < 600 || refundTimeoutDelay > 7 * 24 * 60 * 60) - usage("Locktime (seconds) should be at between 10 minutes and 1 week from now"); - - if (args.length > argIndex) - bitcoinFee = Coin.parseCoin(args[argIndex++]); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - try { - RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); - RepositoryManager.setRepositoryFactory(repositoryFactory); - } catch (DataException e) { - throw new RuntimeException("Repository startup issue: " + e.getMessage()); - } - - try (final Repository repository = RepositoryManager.getRepository()) { - System.out.println("Confirm the following is correct based on the info you've given:"); - - System.out.println(String.format("Refund Bitcoin address: %s", redeemBitcoinAddress)); - System.out.println(String.format("Bitcoin redeem amount: %s", bitcoinAmount.toPlainString())); - - System.out.println(String.format("Redeem Bitcoin address: %s", refundBitcoinAddress)); - System.out.println(String.format("Redeem miner's fee: %s", BTC.FORMAT.format(bitcoinFee))); - - System.out.println(String.format("Redeem script lockTime: %s (%d)", LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC), lockTime)); - System.out.println(String.format("Hash of secret: %s", HashCode.fromBytes(secretHash))); - - System.out.println(String.format("P2SH address: %s", p2shAddress)); - - byte[] redeemScriptBytes = BTCACCT.buildScript(refundBitcoinAddress.getHash(), lockTime, redeemBitcoinAddress.getHash(), secretHash); - System.out.println(String.format("Redeem script: %s", HashCode.fromBytes(redeemScriptBytes))); - - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - Address derivedP2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - - if (!derivedP2shAddress.equals(p2shAddress)) { - System.err.println(String.format("Derived P2SH address %s does not match given address %s", derivedP2shAddress, p2shAddress)); - System.exit(2); - } - - bitcoinAmount = bitcoinAmount.add(bitcoinFee); - - long medianBlockTime = BTC.getInstance().getMedianBlockTime(); - System.out.println(String.format("Median block time: %s", LocalDateTime.ofInstant(Instant.ofEpochSecond(medianBlockTime), ZoneOffset.UTC))); - - long now = System.currentTimeMillis(); - - if (now < medianBlockTime * 1000L) - System.out.println(String.format("Too soon (%s) to redeem based on median block time %s", LocalDateTime.ofInstant(Instant.ofEpochMilli(now), ZoneOffset.UTC), LocalDateTime.ofInstant(Instant.ofEpochSecond(medianBlockTime), ZoneOffset.UTC))); - - // Check P2SH is funded - Coin p2shBalance = BTC.getInstance().getBalance(p2shAddress.toString()); - if (p2shBalance == null) { - System.err.println(String.format("Unable to check P2SH address %s balance", p2shAddress)); - System.exit(2); - } - System.out.println(String.format("P2SH address %s balance: %s", p2shAddress, BTC.FORMAT.format(p2shBalance))); - - // Grab all P2SH funding transactions (just in case there are more than one) - List fundingOutputs = BTC.getInstance().getUnspentOutputs(p2shAddress.toString()); - if (fundingOutputs == null) { - System.err.println(String.format("Can't find outputs for P2SH")); - System.exit(2); - } - - System.out.println(String.format("Found %d output%s for P2SH", fundingOutputs.size(), (fundingOutputs.size() != 1 ? "s" : ""))); - - for (TransactionOutput fundingOutput : fundingOutputs) - System.out.println(String.format("Output %s:%d amount %s", HashCode.fromBytes(fundingOutput.getParentTransactionHash().getBytes()), fundingOutput.getIndex(), BTC.FORMAT.format(fundingOutput.getValue()))); - - if (fundingOutputs.isEmpty()) { - System.err.println(String.format("Can't redeem spent/unfunded P2SH")); - System.exit(2); - } - - if (fundingOutputs.size() != 1) { - System.err.println(String.format("Expecting only one unspent output for P2SH")); - System.exit(2); - } - } catch (DataException e) { - throw new RuntimeException("Repository issue: " + e.getMessage()); - } - } - -} diff --git a/src/test/java/org/qortal/test/btcacct/Common.java b/src/test/java/org/qortal/test/btcacct/Common.java deleted file mode 100644 index 320d1c1cc..000000000 --- a/src/test/java/org/qortal/test/btcacct/Common.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.qortal.test.btcacct; - -import org.bitcoinj.core.Coin; - -public abstract class Common { - - public static final Coin DEFAULT_BTC_FEE = Coin.parseCoin("0.00001000"); - -} diff --git a/src/test/java/org/qortal/test/btcacct/ElectrumXTests.java b/src/test/java/org/qortal/test/btcacct/ElectrumXTests.java deleted file mode 100644 index a8c3cb124..000000000 --- a/src/test/java/org/qortal/test/btcacct/ElectrumXTests.java +++ /dev/null @@ -1,136 +0,0 @@ -package org.qortal.test.btcacct; - -import static org.junit.Assert.*; - -import java.security.Security; -import java.util.List; - -import org.bitcoinj.core.Address; -import org.bitcoinj.params.TestNet3Params; -import org.bitcoinj.script.ScriptBuilder; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; -import org.junit.Test; -import org.qortal.crosschain.ElectrumX; -import org.qortal.utils.BitTwiddling; -import org.qortal.utils.Pair; - -import com.google.common.hash.HashCode; - -public class ElectrumXTests { - - static { - // This must go before any calls to LogManager/Logger - System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); - - Security.insertProviderAt(new BouncyCastleProvider(), 0); - Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); - } - - @Test - public void testInstance() { - ElectrumX electrumX = ElectrumX.getInstance("TEST3"); - assertNotNull(electrumX); - } - - @Test - public void testGetCurrentHeight() { - ElectrumX electrumX = ElectrumX.getInstance("TEST3"); - - Integer height = electrumX.getCurrentHeight(); - - assertNotNull(height); - assertTrue(height > 10000); - System.out.println("Current TEST3 height: " + height); - } - - @Test - public void testGetRecentBlocks() { - ElectrumX electrumX = ElectrumX.getInstance("TEST3"); - - Integer height = electrumX.getCurrentHeight(); - assertNotNull(height); - assertTrue(height > 10000); - - List recentBlockHeaders = electrumX.getBlockHeaders(height - 11, 11); - assertNotNull(recentBlockHeaders); - - System.out.println(String.format("Returned %d recent blocks", recentBlockHeaders.size())); - for (int i = 0; i < recentBlockHeaders.size(); ++i) { - byte[] blockHeader = recentBlockHeaders.get(i); - - // Timestamp(int) is at 4 + 32 + 32 = 68 bytes offset - int offset = 4 + 32 + 32; - int timestamp = BitTwiddling.fromLEBytes(blockHeader, offset); - System.out.println(String.format("Block %d timestamp: %d", height + i, timestamp)); - } - } - - @Test - public void testGetP2PKHBalance() { - ElectrumX electrumX = ElectrumX.getInstance("TEST3"); - - Address address = Address.fromString(TestNet3Params.get(), "n3GNqMveyvaPvUbH469vDRadqpJMPc84JA"); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - Long balance = electrumX.getBalance(script); - - assertNotNull(balance); - assertTrue(balance > 0L); - - System.out.println(String.format("TestNet address %s has balance: %d sats / %d.%08d BTC", address, balance, (balance / 100000000L), (balance % 100000000L))); - } - - @Test - public void testGetP2SHBalance() { - ElectrumX electrumX = ElectrumX.getInstance("TEST3"); - - Address address = Address.fromString(TestNet3Params.get(), "2N4szZUfigj7fSBCEX4PaC8TVbC5EvidaVF"); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - Long balance = electrumX.getBalance(script); - - assertNotNull(balance); - assertTrue(balance > 0L); - - System.out.println(String.format("TestNet address %s has balance: %d sats / %d.%08d BTC", address, balance, (balance / 100000000L), (balance % 100000000L))); - } - - @Test - public void testGetUnspentOutputs() { - ElectrumX electrumX = ElectrumX.getInstance("TEST3"); - - Address address = Address.fromString(TestNet3Params.get(), "2N4szZUfigj7fSBCEX4PaC8TVbC5EvidaVF"); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - List> unspentOutputs = electrumX.getUnspentOutputs(script); - - assertNotNull(unspentOutputs); - assertFalse(unspentOutputs.isEmpty()); - - for (Pair unspentOutput : unspentOutputs) - System.out.println(String.format("TestNet address %s has unspent output at tx %s, output index %d", address, HashCode.fromBytes(unspentOutput.getA()).toString(), unspentOutput.getB())); - } - - @Test - public void testGetRawTransaction() { - ElectrumX electrumX = ElectrumX.getInstance("TEST3"); - - byte[] txHash = HashCode.fromString("7653fea9ffcd829d45ed2672938419a94951b08175982021e77d619b553f29af").asBytes(); - - byte[] rawTransactionBytes = electrumX.getRawTransaction(txHash); - - assertNotNull(rawTransactionBytes); - } - - @Test - public void testGetAddressTransactions() { - ElectrumX electrumX = ElectrumX.getInstance("TEST3"); - - Address address = Address.fromString(TestNet3Params.get(), "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - - List rawTransactions = electrumX.getAddressTransactions(script); - - assertNotNull(rawTransactions); - assertFalse(rawTransactions.isEmpty()); - } - -} diff --git a/src/test/java/org/qortal/test/btcacct/Redeem.java b/src/test/java/org/qortal/test/btcacct/Redeem.java deleted file mode 100644 index 091f22343..000000000 --- a/src/test/java/org/qortal/test/btcacct/Redeem.java +++ /dev/null @@ -1,197 +0,0 @@ -package org.qortal.test.btcacct; - -import java.security.Security; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.Arrays; -import java.util.List; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.ECKey; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.core.Transaction; -import org.bitcoinj.core.TransactionOutput; -import org.bitcoinj.script.Script.ScriptType; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.qortal.controller.Controller; -import org.qortal.crosschain.BTC; -import org.qortal.crosschain.BTCACCT; -import org.qortal.crypto.Crypto; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryFactory; -import org.qortal.repository.RepositoryManager; -import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; -import org.qortal.settings.Settings; - -import com.google.common.hash.HashCode; - -public class Redeem { - - static { - // This must go before any calls to LogManager/Logger - System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); - } - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: Redeem ()")); - System.err.println(String.format("example: Redeem " - + "2NEZboTLhBDPPQciR7sExBhy3TsDi7wV3Cv \\\n" - + "\tmrTDPdM15cFWJC4g223BXX5snicfVJBx6M \\\n" - + "\tec199a4abc9d3bf024349e397535dfee9d287e174aeabae94237eb03a0118c03 \\\n" - + "\t5468697320737472696e672069732065786163746c7920333220627974657321 \\\n" - + "\t1585920000")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length < 5 || args.length > 6) - usage(null); - - Security.insertProviderAt(new BouncyCastleProvider(), 0); - Settings.fileInstance("settings-test.json"); - - BTC btc = BTC.getInstance(); - NetworkParameters params = btc.getNetworkParameters(); - - Address p2shAddress = null; - Address refundBitcoinAddress = null; - byte[] redeemPrivateKey = null; - byte[] secret = null; - int lockTime = 0; - Coin bitcoinFee = Common.DEFAULT_BTC_FEE; - - int argIndex = 0; - try { - p2shAddress = Address.fromString(params, args[argIndex++]); - if (p2shAddress.getOutputScriptType() != ScriptType.P2SH) - usage("P2SH address invalid"); - - refundBitcoinAddress = Address.fromString(params, args[argIndex++]); - if (refundBitcoinAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Refund BTC address must be in P2PKH form"); - - redeemPrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); - // Auto-trim - if (redeemPrivateKey.length >= 37 && redeemPrivateKey.length <= 38) - redeemPrivateKey = Arrays.copyOfRange(redeemPrivateKey, 1, 33); - if (redeemPrivateKey.length != 32) - usage("Redeem private key must be 32 bytes"); - - secret = HashCode.fromString(args[argIndex++]).asBytes(); - if (secret.length == 0) - usage("Invalid secret bytes"); - - lockTime = Integer.parseInt(args[argIndex++]); - - if (args.length > argIndex) - bitcoinFee = Coin.parseCoin(args[argIndex++]); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - try { - RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); - RepositoryManager.setRepositoryFactory(repositoryFactory); - } catch (DataException e) { - throw new RuntimeException("Repository startup issue: " + e.getMessage()); - } - - try (final Repository repository = RepositoryManager.getRepository()) { - System.out.println("Confirm the following is correct based on the info you've given:"); - - System.out.println(String.format("Redeem PRIVATE key: %s", HashCode.fromBytes(redeemPrivateKey))); - System.out.println(String.format("Redeem miner's fee: %s", BTC.FORMAT.format(bitcoinFee))); - System.out.println(String.format("Redeem script lockTime: %s (%d)", LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC), lockTime)); - - // New/derived info - - byte[] secretHash = Crypto.hash160(secret); - System.out.println(String.format("HASH160 of secret: %s", HashCode.fromBytes(secretHash))); - - ECKey redeemKey = ECKey.fromPrivate(redeemPrivateKey); - Address redeemAddress = Address.fromKey(params, redeemKey, ScriptType.P2PKH); - System.out.println(String.format("Redeem recipient (PKH): %s (%s)", redeemAddress, HashCode.fromBytes(redeemAddress.getHash()))); - - System.out.println(String.format("P2SH address: %s", p2shAddress)); - - byte[] redeemScriptBytes = BTCACCT.buildScript(refundBitcoinAddress.getHash(), lockTime, redeemAddress.getHash(), secretHash); - System.out.println(String.format("Redeem script: %s", HashCode.fromBytes(redeemScriptBytes))); - - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - Address derivedP2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - - if (!derivedP2shAddress.equals(p2shAddress)) { - System.err.println(String.format("Derived P2SH address %s does not match given address %s", derivedP2shAddress, p2shAddress)); - System.exit(2); - } - - // Some checks - - System.out.println("\nProcessing:"); - - long medianBlockTime = BTC.getInstance().getMedianBlockTime(); - System.out.println(String.format("Median block time: %s", LocalDateTime.ofInstant(Instant.ofEpochSecond(medianBlockTime), ZoneOffset.UTC))); - - long now = System.currentTimeMillis(); - - if (now < medianBlockTime * 1000L) { - System.err.println(String.format("Too soon (%s) to redeem based on median block time %s", LocalDateTime.ofInstant(Instant.ofEpochMilli(now), ZoneOffset.UTC), LocalDateTime.ofInstant(Instant.ofEpochSecond(medianBlockTime), ZoneOffset.UTC))); - System.exit(2); - } - - // Check P2SH is funded - Coin p2shBalance = BTC.getInstance().getBalance(p2shAddress.toString()); - if (p2shBalance == null) { - System.err.println(String.format("Unable to check P2SH address %s balance", p2shAddress)); - System.exit(2); - } - System.out.println(String.format("P2SH address %s balance: %s BTC", p2shAddress, p2shBalance.toPlainString())); - - // Grab all P2SH funding transactions (just in case there are more than one) - List fundingOutputs = BTC.getInstance().getUnspentOutputs(p2shAddress.toString()); - if (fundingOutputs == null) { - System.err.println(String.format("Can't find outputs for P2SH")); - System.exit(2); - } - - System.out.println(String.format("Found %d output%s for P2SH", fundingOutputs.size(), (fundingOutputs.size() != 1 ? "s" : ""))); - - for (TransactionOutput fundingOutput : fundingOutputs) - System.out.println(String.format("Output %s:%d amount %s", HashCode.fromBytes(fundingOutput.getParentTransactionHash().getBytes()), fundingOutput.getIndex(), BTC.FORMAT.format(fundingOutput.getValue()))); - - if (fundingOutputs.isEmpty()) { - System.err.println(String.format("Can't redeem spent/unfunded P2SH")); - System.exit(2); - } - - if (fundingOutputs.size() != 1) { - System.err.println(String.format("Expecting only one unspent output for P2SH")); - // No longer fatal - } - - for (TransactionOutput fundingOutput : fundingOutputs) - System.out.println(String.format("Using output %s:%d for redeem", HashCode.fromBytes(fundingOutput.getParentTransactionHash().getBytes()), fundingOutput.getIndex())); - - Coin redeemAmount = p2shBalance.subtract(bitcoinFee); - System.out.println(String.format("Spending %s of output, with %s as mining fee", BTC.FORMAT.format(redeemAmount), BTC.FORMAT.format(bitcoinFee))); - - Transaction redeemTransaction = BTCACCT.buildRedeemTransaction(redeemAmount, redeemKey, fundingOutputs, redeemScriptBytes, secret); - - byte[] redeemBytes = redeemTransaction.bitcoinSerialize(); - - System.out.println(String.format("\nLoad this transaction into your wallet and broadcast:\n%s\n", HashCode.fromBytes(redeemBytes).toString())); - } catch (NumberFormatException e) { - usage(String.format("Number format exception: %s", e.getMessage())); - } catch (DataException e) { - throw new RuntimeException("Repository issue: " + e.getMessage()); - } - } - -} diff --git a/src/test/java/org/qortal/test/btcacct/Refund.java b/src/test/java/org/qortal/test/btcacct/Refund.java deleted file mode 100644 index a694ee145..000000000 --- a/src/test/java/org/qortal/test/btcacct/Refund.java +++ /dev/null @@ -1,201 +0,0 @@ -package org.qortal.test.btcacct; - -import java.security.Security; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.Arrays; -import java.util.List; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.ECKey; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.core.Transaction; -import org.bitcoinj.core.TransactionOutput; -import org.bitcoinj.script.Script.ScriptType; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.qortal.controller.Controller; -import org.qortal.crosschain.BTC; -import org.qortal.crosschain.BTCACCT; -import org.qortal.crypto.Crypto; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryFactory; -import org.qortal.repository.RepositoryManager; -import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; -import org.qortal.settings.Settings; - -import com.google.common.hash.HashCode; - -public class Refund { - - static { - // This must go before any calls to LogManager/Logger - System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); - } - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: Refund ()")); - System.err.println(String.format("example: Refund " - + "2NEZboTLhBDPPQciR7sExBhy3TsDi7wV3Cv \\\n" - + "\tef027fb5828c5e201eaf6de4cd3b0b340d16a191ef848cd691f35ef8f727358c9c01b576fb7e \\\n" - + "\tn2N5VKrzq39nmuefZwp3wBiF4icdXX2B6o \\\n" - + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" - + "\t1585920000")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length < 5 || args.length > 6) - usage(null); - - Security.insertProviderAt(new BouncyCastleProvider(), 0); - Settings.fileInstance("settings-test.json"); - - BTC btc = BTC.getInstance(); - NetworkParameters params = btc.getNetworkParameters(); - - Address p2shAddress = null; - byte[] refundPrivateKey = null; - Address redeemBitcoinAddress = null; - byte[] secretHash = null; - int lockTime = 0; - Coin bitcoinFee = Common.DEFAULT_BTC_FEE; - - int argIndex = 0; - try { - p2shAddress = Address.fromString(params, args[argIndex++]); - if (p2shAddress.getOutputScriptType() != ScriptType.P2SH) - usage("P2SH address invalid"); - - refundPrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); - // Auto-trim - if (refundPrivateKey.length >= 37 && refundPrivateKey.length <= 38) - refundPrivateKey = Arrays.copyOfRange(refundPrivateKey, 1, 33); - if (refundPrivateKey.length != 32) - usage("Refund private key must be 32 bytes"); - - redeemBitcoinAddress = Address.fromString(params, args[argIndex++]); - if (redeemBitcoinAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Their BTC address must be in P2PKH form"); - - secretHash = HashCode.fromString(args[argIndex++]).asBytes(); - if (secretHash.length != 20) - usage("HASH160 of secret must be 20 bytes"); - - lockTime = Integer.parseInt(args[argIndex++]); - - if (args.length > argIndex) - bitcoinFee = Coin.parseCoin(args[argIndex++]); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - try { - RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); - RepositoryManager.setRepositoryFactory(repositoryFactory); - } catch (DataException e) { - throw new RuntimeException("Repository startup issue: " + e.getMessage()); - } - - try (final Repository repository = RepositoryManager.getRepository()) { - System.out.println("Confirm the following is correct based on the info you've given:"); - - System.out.println(String.format("Refund PRIVATE key: %s", HashCode.fromBytes(refundPrivateKey))); - System.out.println(String.format("Redeem Bitcoin address: %s", redeemBitcoinAddress)); - System.out.println(String.format("Redeem script lockTime: %s (%d)", LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC), lockTime)); - System.out.println(String.format("P2SH address: %s", p2shAddress)); - System.out.println(String.format("Refund miner's fee: %s", BTC.FORMAT.format(bitcoinFee))); - - // New/derived info - - System.out.println("\nCHECKING info from other party:"); - - ECKey refundKey = ECKey.fromPrivate(refundPrivateKey); - Address refundAddress = Address.fromKey(params, refundKey, ScriptType.P2PKH); - System.out.println(String.format("Refund recipient (PKH): %s (%s)", refundAddress, HashCode.fromBytes(refundAddress.getHash()))); - - byte[] redeemScriptBytes = BTCACCT.buildScript(refundAddress.getHash(), lockTime, redeemBitcoinAddress.getHash(), secretHash); - System.out.println(String.format("Redeem script: %s", HashCode.fromBytes(redeemScriptBytes))); - - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - Address derivedP2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - - if (!derivedP2shAddress.equals(p2shAddress)) { - System.err.println(String.format("Derived P2SH address %s does not match given address %s", derivedP2shAddress, p2shAddress)); - System.exit(2); - } - - // Some checks - - System.out.println("\nProcessing:"); - - long medianBlockTime = BTC.getInstance().getMedianBlockTime(); - System.out.println(String.format("Median block time: %s", LocalDateTime.ofInstant(Instant.ofEpochSecond(medianBlockTime), ZoneOffset.UTC))); - - long now = System.currentTimeMillis(); - - if (now < medianBlockTime * 1000L) { - System.err.println(String.format("Too soon (%s) to refund based on median block time %s", LocalDateTime.ofInstant(Instant.ofEpochMilli(now), ZoneOffset.UTC), LocalDateTime.ofInstant(Instant.ofEpochSecond(medianBlockTime), ZoneOffset.UTC))); - System.exit(2); - } - - if (now < lockTime * 1000L) { - System.err.println(String.format("Too soon (%s) to refund based on lockTime %s", LocalDateTime.ofInstant(Instant.ofEpochMilli(now), ZoneOffset.UTC), LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC))); - System.exit(2); - } - - // Check P2SH is funded - Coin p2shBalance = BTC.getInstance().getBalance(p2shAddress.toString()); - if (p2shBalance == null) { - System.err.println(String.format("Unable to check P2SH address %s balance", p2shAddress)); - System.exit(2); - } - System.out.println(String.format("P2SH address %s balance: %s BTC", p2shAddress, p2shBalance.toPlainString())); - - // Grab all P2SH funding transactions (just in case there are more than one) - List fundingOutputs = BTC.getInstance().getUnspentOutputs(p2shAddress.toString()); - if (fundingOutputs == null) { - System.err.println(String.format("Can't find outputs for P2SH")); - System.exit(2); - } - - System.out.println(String.format("Found %d output%s for P2SH", fundingOutputs.size(), (fundingOutputs.size() != 1 ? "s" : ""))); - - for (TransactionOutput fundingOutput : fundingOutputs) - System.out.println(String.format("Output %s:%d amount %s", HashCode.fromBytes(fundingOutput.getParentTransactionHash().getBytes()), fundingOutput.getIndex(), BTC.FORMAT.format(fundingOutput.getValue()))); - - if (fundingOutputs.isEmpty()) { - System.err.println(String.format("Can't refund spent/unfunded P2SH")); - System.exit(2); - } - - if (fundingOutputs.size() != 1) { - System.err.println(String.format("Expecting only one unspent output for P2SH")); - // No longer fatal - } - - for (TransactionOutput fundingOutput : fundingOutputs) - System.out.println(String.format("Using output %s:%d for redeem", HashCode.fromBytes(fundingOutput.getParentTransactionHash().getBytes()), fundingOutput.getIndex())); - - Coin refundAmount = p2shBalance.subtract(bitcoinFee); - System.out.println(String.format("Spending %s of output, with %s as mining fee", BTC.FORMAT.format(refundAmount), BTC.FORMAT.format(bitcoinFee))); - - Transaction redeemTransaction = BTCACCT.buildRefundTransaction(refundAmount, refundKey, fundingOutputs, redeemScriptBytes, lockTime); - - byte[] redeemBytes = redeemTransaction.bitcoinSerialize(); - - System.out.println(String.format("\nLoad this transaction into your wallet and broadcast:\n%s\n", HashCode.fromBytes(redeemBytes).toString())); - } catch (NumberFormatException e) { - usage(String.format("Number format exception: %s", e.getMessage())); - } catch (DataException e) { - throw new RuntimeException("Repository issue: " + e.getMessage()); - } - } - -} diff --git a/src/test/java/org/qortal/test/common/AccountUtils.java b/src/test/java/org/qortal/test/common/AccountUtils.java index 59722ae1e..0e7ef0204 100644 --- a/src/test/java/org/qortal/test/common/AccountUtils.java +++ b/src/test/java/org/qortal/test/common/AccountUtils.java @@ -20,15 +20,19 @@ public class AccountUtils { public static final int txGroupId = Group.NO_GROUP; public static final long fee = 1L * Amounts.MULTIPLIER; - public static void pay(Repository repository, String sender, String recipient, long amount) throws DataException { - PrivateKeyAccount sendingAccount = Common.getTestAccount(repository, sender); - PrivateKeyAccount recipientAccount = Common.getTestAccount(repository, recipient); + public static void pay(Repository repository, String testSenderName, String testRecipientName, long amount) throws DataException { + PrivateKeyAccount sendingAccount = Common.getTestAccount(repository, testSenderName); + PrivateKeyAccount recipientAccount = Common.getTestAccount(repository, testRecipientName); + + pay(repository, sendingAccount, recipientAccount.getAddress(), amount); + } + public static void pay(Repository repository, PrivateKeyAccount sendingAccount, String recipientAddress, long amount) throws DataException { byte[] reference = sendingAccount.getLastReference(); long timestamp = repository.getTransactionRepository().fromSignature(reference).getTimestamp() + 1; BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, sendingAccount.getPublicKey(), fee, null); - TransactionData transactionData = new PaymentTransactionData(baseTransactionData, recipientAccount.getAddress(), amount); + TransactionData transactionData = new PaymentTransactionData(baseTransactionData, recipientAddress, amount); TransactionUtils.signAndMint(repository, transactionData, sendingAccount); } diff --git a/src/test/java/org/qortal/test/common/ApiCommon.java b/src/test/java/org/qortal/test/common/ApiCommon.java index 25e6ec530..b5d02b24c 100644 --- a/src/test/java/org/qortal/test/common/ApiCommon.java +++ b/src/test/java/org/qortal/test/common/ApiCommon.java @@ -1,16 +1,30 @@ package org.qortal.test.common; +import static org.junit.Assert.*; + import java.lang.reflect.Field; import org.eclipse.jetty.server.Request; import org.junit.Before; +import org.qortal.api.ApiError; +import org.qortal.api.ApiException; import org.qortal.repository.DataException; public class ApiCommon extends Common { + public static final long MAX_API_RESPONSE_PERIOD = 2_000L; // ms + public static final Boolean[] ALL_BOOLEAN_VALUES = new Boolean[] { null, true, false }; public static final Boolean[] TF_BOOLEAN_VALUES = new Boolean[] { true, false }; + public static final Integer[] SAMPLE_LIMIT_VALUES = new Integer[] { null, 0, 1, 20 }; + public static final Integer[] SAMPLE_OFFSET_VALUES = new Integer[] { null, 0, 1, 5 }; + + @FunctionalInterface + public interface SlicedApiCall { + public abstract void call(Integer limit, Integer offset, Boolean reverse); + } + public static class FakeRequest extends Request { public FakeRequest() { super(null, null); @@ -48,4 +62,50 @@ public static Object buildResource(Class resourceClass) { } } + public static void assertApiError(ApiError expectedApiError, Runnable apiCall, Long maxResponsePeriod) { + try { + long beforeTimestamp = System.currentTimeMillis(); + apiCall.run(); + + if (maxResponsePeriod != null) { + long responsePeriod = System.currentTimeMillis() - beforeTimestamp; + if (responsePeriod > maxResponsePeriod) + fail(String.format("API call response period %d ms greater than max allowed (%d ms)", responsePeriod, maxResponsePeriod)); + } + } catch (ApiException e) { + ApiError actualApiError = ApiError.fromCode(e.error); + assertEquals(expectedApiError, actualApiError); + } + } + + public static void assertApiError(ApiError expectedApiError, Runnable apiCall) { + assertApiError(expectedApiError, apiCall, MAX_API_RESPONSE_PERIOD); + } + + public static void assertNoApiError(Runnable apiCall, Long maxResponsePeriod) { + try { + long beforeTimestamp = System.currentTimeMillis(); + apiCall.run(); + + if (maxResponsePeriod != null) { + long responsePeriod = System.currentTimeMillis() - beforeTimestamp; + if (responsePeriod > maxResponsePeriod) + fail(String.format("API call response period %d ms greater than max allowed (%d ms)", responsePeriod, maxResponsePeriod)); + } + } catch (ApiException e) { + fail("ApiException unexpected"); + } + } + + public static void assertNoApiError(Runnable apiCall) { + assertNoApiError(apiCall, MAX_API_RESPONSE_PERIOD); + } + + public static void assertNoApiError(SlicedApiCall apiCall) { + for (Integer limit : SAMPLE_LIMIT_VALUES) + for (Integer offset : SAMPLE_OFFSET_VALUES) + for (Boolean reverse : ALL_BOOLEAN_VALUES) + assertNoApiError(() -> apiCall.call(limit, offset, reverse)); + } + } diff --git a/src/test/java/org/qortal/test/common/ArbitraryUtils.java b/src/test/java/org/qortal/test/common/ArbitraryUtils.java new file mode 100644 index 000000000..5a67ccae3 --- /dev/null +++ b/src/test/java/org/qortal/test/common/ArbitraryUtils.java @@ -0,0 +1,89 @@ +package org.qortal.test.common; + +import org.qortal.account.PrivateKeyAccount; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataTransactionBuilder; +import org.qortal.arbitrary.misc.Service; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.Transaction; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Random; + +import static org.junit.Assert.assertEquals; + +public class ArbitraryUtils { + + public static ArbitraryDataFile createAndMintTxn(Repository repository, String publicKey58, Path path, String name, String identifier, + ArbitraryTransactionData.Method method, Service service, PrivateKeyAccount account, + int chunkSize) throws DataException { + + ArbitraryDataTransactionBuilder txnBuilder = new ArbitraryDataTransactionBuilder( + repository, publicKey58, path, name, method, service, identifier); + + txnBuilder.setChunkSize(chunkSize); + txnBuilder.build(); + txnBuilder.computeNonce(); + ArbitraryTransactionData transactionData = txnBuilder.getArbitraryTransactionData(); + Transaction.ValidationResult result = TransactionUtils.signAndImport(repository, transactionData, account); + assertEquals(Transaction.ValidationResult.OK, result); + BlockUtils.mintBlock(repository); + + // We need a new ArbitraryDataFile instance because the files will have been moved to the signature's folder + byte[] hash = txnBuilder.getArbitraryDataFile().getHash(); + byte[] signature = transactionData.getSignature(); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); + arbitraryDataFile.setMetadataHash(transactionData.getMetadataHash()); + + return arbitraryDataFile; + } + + public static ArbitraryDataFile createAndMintTxn(Repository repository, String publicKey58, Path path, String name, String identifier, + ArbitraryTransactionData.Method method, Service service, PrivateKeyAccount account) throws DataException { + + // Use default chunk size + int chunkSize = ArbitraryDataFile.CHUNK_SIZE; + return ArbitraryUtils.createAndMintTxn(repository, publicKey58, path, name, identifier, method, service, account, chunkSize); + } + + public static Path generateRandomDataPath(int length) throws IOException { + // Create a file in a random temp directory + Path tempDir = Files.createTempDirectory("generateRandomDataPath"); + File file = new File(Paths.get(tempDir.toString(), "file.txt").toString()); + file.deleteOnExit(); + + // Write a random string to the file + BufferedWriter file1Writer = new BufferedWriter(new FileWriter(file)); + String initialString = ArbitraryUtils.generateRandomString(length - 1); // -1 due to newline at EOF + + // Add a newline every 50 chars + // initialString = initialString.replaceAll("(.{50})", "$1\n"); + + file1Writer.write(initialString); + file1Writer.newLine(); + file1Writer.close(); + + return tempDir; + } + + public static String generateRandomString(int length) { + int leftLimit = 48; // numeral '0' + int rightLimit = 122; // letter 'z' + Random random = new Random(); + + return random.ints(leftLimit, rightLimit + 1) + .filter(i -> (i <= 57 || i >= 65) && (i <= 90 || i >= 97)) + .limit(length) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + } + +} diff --git a/src/test/java/org/qortal/test/common/AtUtils.java b/src/test/java/org/qortal/test/common/AtUtils.java new file mode 100644 index 000000000..3bc2b235b --- /dev/null +++ b/src/test/java/org/qortal/test/common/AtUtils.java @@ -0,0 +1,81 @@ +package org.qortal.test.common; + +import org.ciyam.at.CompilationException; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.DeployAtTransaction; + +import java.nio.ByteBuffer; + +public class AtUtils { + + public static byte[] buildSimpleAT() { + // Pretend we use 4 values in data segment + int addrCounter = 4; + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(512); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile AT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + public static DeployAtTransaction doDeployAT(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "Test AT"; + String description = "Test AT"; + String atType = "Test"; + String tags = "TEST"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } +} diff --git a/src/test/java/org/qortal/test/common/BlockUtils.java b/src/test/java/org/qortal/test/common/BlockUtils.java index 82c41d501..3077b65b8 100644 --- a/src/test/java/org/qortal/test/common/BlockUtils.java +++ b/src/test/java/org/qortal/test/common/BlockUtils.java @@ -5,7 +5,7 @@ import org.qortal.account.PrivateKeyAccount; import org.qortal.block.Block; import org.qortal.block.BlockChain; -import org.qortal.block.BlockMinter; +import org.qortal.controller.BlockMinter; import org.qortal.data.block.BlockData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; diff --git a/src/test/java/org/qortal/test/common/Common.java b/src/test/java/org/qortal/test/common/Common.java index 24c86690b..c45fcfd70 100644 --- a/src/test/java/org/qortal/test/common/Common.java +++ b/src/test/java/org/qortal/test/common/Common.java @@ -2,8 +2,11 @@ import static org.junit.Assert.*; +import java.io.IOException; import java.math.BigDecimal; import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.Security; import java.util.ArrayList; import java.util.Collections; @@ -15,6 +18,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bouncycastle.jce.provider.BouncyCastleProvider; @@ -46,9 +50,15 @@ public class Common { private static final Logger LOGGER = LogManager.getLogger(Common.class); - public static final String testConnectionUrl = "jdbc:hsqldb:mem:testdb"; - // For debugging, use this instead to write DB to disk for examination: - // public static final String testConnectionUrl = "jdbc:hsqldb:file:testdb/blockchain;create=true"; + public static final String testConnectionUrlMemory = "jdbc:hsqldb:mem:testdb"; + public static final String testConnectionUrlDisk = "jdbc:hsqldb:file:%s/blockchain;create=true"; + + // For debugging, use testConnectionUrlDisk instead of memory, to write DB to disk for examination. + // This can be achieved using `Common.useSettingsAndDb(Common.testSettingsFilename, false);` + // where `false` specifies to use a repository on disk rather than one in memory. + // Make sure to also comment out `Common.deleteTestRepository();` in closeRepository() below, so that + // the files remain after the test finishes. + public static final String testSettingsFilename = "test-settings-v2.json"; @@ -100,7 +110,7 @@ public static List getTestAccounts(Repository repository) { return testAccountsByName.values().stream().map(account -> new TestAccount(repository, account)).collect(Collectors.toList()); } - public static void useSettings(String settingsFilename) throws DataException { + public static void useSettingsAndDb(String settingsFilename, boolean dbInMemory) throws DataException { closeRepository(); // Load/check settings, which potentially sets up blockchain config, etc. @@ -109,11 +119,15 @@ public static void useSettings(String settingsFilename) throws DataException { assertNotNull("Test settings JSON file not found", testSettingsUrl); Settings.fileInstance(testSettingsUrl.getPath()); - setRepository(); + setRepository(dbInMemory); resetBlockchain(); } + public static void useSettings(String settingsFilename) throws DataException { + Common.useSettingsAndDb(settingsFilename, true); + } + public static void useDefaultSettings() throws DataException { useSettings(testSettingsFilename); NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset()); @@ -186,15 +200,33 @@ private static void checkOrphanedLists(String typeName, List initial, Lis assertTrue(String.format("Non-genesis %s remains", typeName), remainingClone.isEmpty()); } - @BeforeClass - public static void setRepository() throws DataException { - RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(testConnectionUrl); + public static void setRepository(boolean inMemory) throws DataException { + String connectionUrlDisk = String.format(testConnectionUrlDisk, Settings.getInstance().getRepositoryPath()); + String connectionUrl = inMemory ? testConnectionUrlMemory : connectionUrlDisk; + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(connectionUrl); RepositoryManager.setRepositoryFactory(repositoryFactory); } + public static void deleteTestRepository() throws DataException { + // Delete repository directory if exists + Path repositoryPath = Paths.get(Settings.getInstance().getRepositoryPath()); + try { + FileUtils.deleteDirectory(repositoryPath.toFile()); + } catch (IOException e) { + throw new DataException(String.format("Unable to delete test repository: %s", e.getMessage())); + } + } + + @BeforeClass + public static void setRepositoryInMemory() throws DataException { + Common.deleteTestRepository(); + Common.setRepository(true); + } + @AfterClass public static void closeRepository() throws DataException { RepositoryManager.closeRepositoryFactory(); + Common.deleteTestRepository(); // Comment out this line in you need to inspect the database after running a test } // Test assertions diff --git a/src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java b/src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java index 0b48748df..d831eaf13 100644 --- a/src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java +++ b/src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java @@ -4,6 +4,7 @@ import java.util.List; import org.qortal.account.PrivateKeyAccount; +import org.qortal.arbitrary.misc.Service; import org.qortal.asset.Asset; import org.qortal.data.PaymentData; import org.qortal.data.transaction.ArbitraryTransactionData; @@ -16,8 +17,21 @@ public class ArbitraryTestTransaction extends TestTransaction { public static TransactionData randomTransaction(Repository repository, PrivateKeyAccount account, boolean wantValid) throws DataException { - final int version = 4; - final int service = 123; + final int version = 5; + final Service service = Service.ARBITRARY_DATA; + final int nonce = 0; + final int size = 4 * 1024 * 1024; + final String name = "TEST"; + final String identifier = "qortal_avatar"; + final ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT; + + final byte[] secret = new byte[32]; + random.nextBytes(secret); + + final ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP; + + final byte[] metadataHash = new byte[32]; + random.nextBytes(metadataHash); byte[] data = new byte[1024]; random.nextBytes(data); @@ -31,7 +45,8 @@ public static TransactionData randomTransaction(Repository repository, PrivateKe List payments = new ArrayList<>(); payments.add(new PaymentData(recipient, assetId, amount)); - return new ArbitraryTransactionData(generateBase(account), version, service, data, dataType, payments); + return new ArbitraryTransactionData(generateBase(account), version, service, nonce, size,name, identifier, + method, secret, compression, data, dataType, metadataHash, payments); } } diff --git a/src/test/java/org/qortal/test/common/transaction/PresenceTestTransaction.java b/src/test/java/org/qortal/test/common/transaction/PresenceTestTransaction.java new file mode 100644 index 000000000..64df87f4f --- /dev/null +++ b/src/test/java/org/qortal/test/common/transaction/PresenceTestTransaction.java @@ -0,0 +1,25 @@ +package org.qortal.test.common.transaction; + +import com.google.common.primitives.Longs; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.PresenceTransaction.PresenceType; +import org.qortal.utils.NTP; + +public class PresenceTestTransaction extends TestTransaction { + + public static TransactionData randomTransaction(Repository repository, PrivateKeyAccount account, boolean wantValid) throws DataException { + final int nonce = 0; + + byte[] tradePrivateKey = new byte[32]; + PrivateKeyAccount tradeNativeAccount = new PrivateKeyAccount(repository, tradePrivateKey); + long timestamp = NTP.getTime(); + byte[] timestampSignature = tradeNativeAccount.sign(Longs.toByteArray(timestamp)); + + return new PresenceTransactionData(generateBase(account), nonce, PresenceType.TRADE_BOT, timestampSignature); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/BitcoinTests.java b/src/test/java/org/qortal/test/crosschain/BitcoinTests.java new file mode 100644 index 000000000..af879e083 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/BitcoinTests.java @@ -0,0 +1,115 @@ +package org.qortal.test.crosschain; + +import static org.junit.Assert.*; + +import java.util.Arrays; + +import org.bitcoinj.core.Transaction; +import org.bitcoinj.store.BlockStoreException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.repository.DataException; +import org.qortal.test.common.Common; + +public class BitcoinTests extends Common { + + private Bitcoin bitcoin; + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); // TestNet3 + bitcoin = Bitcoin.getInstance(); + } + + @After + public void afterTest() { + Bitcoin.resetForTesting(); + bitcoin = null; + } + + @Test + public void testGetMedianBlockTime() throws BlockStoreException, ForeignBlockchainException { + System.out.println(String.format("Starting BTC instance...")); + System.out.println(String.format("BTC instance started")); + + long before = System.currentTimeMillis(); + System.out.println(String.format("Bitcoin median blocktime: %d", bitcoin.getMedianBlockTime())); + long afterFirst = System.currentTimeMillis(); + + System.out.println(String.format("Bitcoin median blocktime: %d", bitcoin.getMedianBlockTime())); + long afterSecond = System.currentTimeMillis(); + + long firstPeriod = afterFirst - before; + long secondPeriod = afterSecond - afterFirst; + + System.out.println(String.format("1st call: %d ms, 2nd call: %d ms", firstPeriod, secondPeriod)); + + assertTrue("2nd call should be quicker than 1st", secondPeriod < firstPeriod); + assertTrue("2nd call should take less than 5 seconds", secondPeriod < 5000L); + } + + @Test + public void testFindHtlcSecret() throws ForeignBlockchainException { + // This actually exists on TEST3 but can take a while to fetch + String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + + byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); + byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); + + assertNotNull(secret); + assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret)); + } + + @Test + public void testBuildSpend() { + String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; + + String recipient = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + long amount = 1000L; + + Transaction transaction = bitcoin.buildSpend(xprv58, recipient, amount); + assertNotNull(transaction); + + // Check spent key caching doesn't affect outcome + + transaction = bitcoin.buildSpend(xprv58, recipient, amount); + assertNotNull(transaction); + } + + @Test + public void testGetWalletBalance() { + String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; + + Long balance = bitcoin.getWalletBalance(xprv58); + + assertNotNull(balance); + + System.out.println(bitcoin.format(balance)); + + // Check spent key caching doesn't affect outcome + + Long repeatBalance = bitcoin.getWalletBalance(xprv58); + + assertNotNull(repeatBalance); + + System.out.println(bitcoin.format(repeatBalance)); + + assertEquals(balance, repeatBalance); + } + + @Test + public void testGetUnusedReceiveAddress() throws ForeignBlockchainException { + String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; + + String address = bitcoin.getUnusedReceiveAddress(xprv58); + + assertNotNull(address); + + System.out.println(address); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/DogecoinTests.java b/src/test/java/org/qortal/test/crosschain/DogecoinTests.java new file mode 100644 index 000000000..2b0410c35 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/DogecoinTests.java @@ -0,0 +1,115 @@ +package org.qortal.test.crosschain; + +import org.bitcoinj.core.Transaction; +import org.bitcoinj.store.BlockStoreException; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Dogecoin; +import org.qortal.repository.DataException; +import org.qortal.test.common.Common; + +import java.util.Arrays; + +import static org.junit.Assert.*; + +public class DogecoinTests extends Common { + + private Dogecoin dogecoin; + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); // TestNet3 + dogecoin = Dogecoin.getInstance(); + } + + @After + public void afterTest() { + Dogecoin.resetForTesting(); + dogecoin = null; + } + + @Test + public void testGetMedianBlockTime() throws BlockStoreException, ForeignBlockchainException { + long before = System.currentTimeMillis(); + System.out.println(String.format("Dogecoin median blocktime: %d", dogecoin.getMedianBlockTime())); + long afterFirst = System.currentTimeMillis(); + + System.out.println(String.format("Dogecoin median blocktime: %d", dogecoin.getMedianBlockTime())); + long afterSecond = System.currentTimeMillis(); + + long firstPeriod = afterFirst - before; + long secondPeriod = afterSecond - afterFirst; + + System.out.println(String.format("1st call: %d ms, 2nd call: %d ms", firstPeriod, secondPeriod)); + + assertTrue("2nd call should be quicker than 1st", secondPeriod < firstPeriod); + assertTrue("2nd call should take less than 5 seconds", secondPeriod < 5000L); + } + + @Test + @Ignore(value = "Doesn't work, to be fixed later") + public void testFindHtlcSecret() throws ForeignBlockchainException { + // This actually exists on TEST3 but can take a while to fetch + String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + + byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); + byte[] secret = BitcoinyHTLC.findHtlcSecret(dogecoin, p2shAddress); + + assertNotNull("secret not found", secret); + assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret)); + } + + @Test + @Ignore(value = "No testnet nodes available, so we can't regularly test buildSpend yet") + public void testBuildSpend() { + String xprv58 = "dgpv51eADS3spNJh9drNeW1Tc1P9z2LyaQRXPBortsq6yice1k47C2u2Prvgxycr2ihNBWzKZ2LthcBBGiYkWZ69KUTVkcLVbnjq7pD8mnApEru"; + + String recipient = "DP1iFao33xdEPa5vaArpj7sykfzKNeiJeX"; + long amount = 1000L; + + Transaction transaction = dogecoin.buildSpend(xprv58, recipient, amount); + assertNotNull("insufficient funds", transaction); + + // Check spent key caching doesn't affect outcome + + transaction = dogecoin.buildSpend(xprv58, recipient, amount); + assertNotNull("insufficient funds", transaction); + } + + @Test + public void testGetWalletBalance() { + String xprv58 = "dgpv51eADS3spNJh9drNeW1Tc1P9z2LyaQRXPBortsq6yice1k47C2u2Prvgxycr2ihNBWzKZ2LthcBBGiYkWZ69KUTVkcLVbnjq7pD8mnApEru"; + + Long balance = dogecoin.getWalletBalance(xprv58); + + assertNotNull(balance); + + System.out.println(dogecoin.format(balance)); + + // Check spent key caching doesn't affect outcome + + Long repeatBalance = dogecoin.getWalletBalance(xprv58); + + assertNotNull(repeatBalance); + + System.out.println(dogecoin.format(repeatBalance)); + + assertEquals(balance, repeatBalance); + } + + @Test + public void testGetUnusedReceiveAddress() throws ForeignBlockchainException { + String xprv58 = "dgpv51eADS3spNJh9drNeW1Tc1P9z2LyaQRXPBortsq6yice1k47C2u2Prvgxycr2ihNBWzKZ2LthcBBGiYkWZ69KUTVkcLVbnjq7pD8mnApEru"; + + String address = dogecoin.getUnusedReceiveAddress(xprv58); + + assertNotNull(address); + + System.out.println(address); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/ElectrumXTests.java b/src/test/java/org/qortal/test/crosschain/ElectrumXTests.java new file mode 100644 index 000000000..b7e57cf38 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/ElectrumXTests.java @@ -0,0 +1,201 @@ +package org.qortal.test.crosschain; + +import static org.junit.Assert.*; + +import java.security.Security; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; + +import org.bitcoinj.core.Address; +import org.bitcoinj.params.TestNet3Params; +import org.bitcoinj.script.ScriptBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.junit.Test; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.BitcoinyTransaction; +import org.qortal.crosschain.ElectrumX; +import org.qortal.crosschain.TransactionHash; +import org.qortal.crosschain.UnspentOutput; +import org.qortal.crosschain.Bitcoin.BitcoinNet; +import org.qortal.crosschain.ElectrumX.Server.ConnectionType; +import org.qortal.utils.BitTwiddling; + +import com.google.common.hash.HashCode; + +public class ElectrumXTests { + + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + } + + private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ElectrumX.Server.ConnectionType.class); + static { + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.TCP, 50001); + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.SSL, 50002); + } + + private ElectrumX getInstance() { + return new ElectrumX("Bitcoin-" + BitcoinNet.TEST3.name(), BitcoinNet.TEST3.getGenesisHash(), BitcoinNet.TEST3.getServers(), DEFAULT_ELECTRUMX_PORTS); + } + + @Test + public void testInstance() { + ElectrumX electrumX = getInstance(); + assertNotNull(electrumX); + } + + @Test + public void testGetCurrentHeight() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + int height = electrumX.getCurrentHeight(); + + assertTrue(height > 10000); + System.out.println("Current TEST3 height: " + height); + } + + @Test + public void testInvalidRequest() { + ElectrumX electrumX = getInstance(); + try { + electrumX.getRawBlockHeaders(-1, -1); + } catch (ForeignBlockchainException e) { + // Should throw due to negative start block height + return; + } + + fail("Negative start block height should cause error"); + } + + @Test + public void testGetRecentBlocks() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + int height = electrumX.getCurrentHeight(); + assertTrue(height > 10000); + + List recentBlockHeaders = electrumX.getRawBlockHeaders(height - 11, 11); + + System.out.println(String.format("Returned %d recent blocks", recentBlockHeaders.size())); + for (int i = 0; i < recentBlockHeaders.size(); ++i) { + byte[] blockHeader = recentBlockHeaders.get(i); + + // Timestamp(int) is at 4 + 32 + 32 = 68 bytes offset + int offset = 4 + 32 + 32; + int timestamp = BitTwiddling.intFromLEBytes(blockHeader, offset); + System.out.println(String.format("Block %d timestamp: %d", height + i, timestamp)); + } + } + + @Test + public void testGetP2PKHBalance() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + Address address = Address.fromString(TestNet3Params.get(), "n3GNqMveyvaPvUbH469vDRadqpJMPc84JA"); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + long balance = electrumX.getConfirmedBalance(script); + + assertTrue(balance > 0L); + + System.out.println(String.format("TestNet address %s has balance: %d sats / %d.%08d BTC", address, balance, (balance / 100000000L), (balance % 100000000L))); + } + + @Test + public void testGetP2SHBalance() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + Address address = Address.fromString(TestNet3Params.get(), "2N4szZUfigj7fSBCEX4PaC8TVbC5EvidaVF"); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + long balance = electrumX.getConfirmedBalance(script); + + assertTrue(balance > 0L); + + System.out.println(String.format("TestNet address %s has balance: %d sats / %d.%08d BTC", address, balance, (balance / 100000000L), (balance % 100000000L))); + } + + @Test + public void testGetUnspentOutputs() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + Address address = Address.fromString(TestNet3Params.get(), "2N4szZUfigj7fSBCEX4PaC8TVbC5EvidaVF"); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + List unspentOutputs = electrumX.getUnspentOutputs(script, false); + + assertFalse(unspentOutputs.isEmpty()); + + for (UnspentOutput unspentOutput : unspentOutputs) + System.out.println(String.format("TestNet address %s has unspent output at tx %s, output index %d", address, HashCode.fromBytes(unspentOutput.hash), unspentOutput.index)); + } + + @Test + public void testGetRawTransaction() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + byte[] txHash = HashCode.fromString("7653fea9ffcd829d45ed2672938419a94951b08175982021e77d619b553f29af").asBytes(); + + byte[] rawTransactionBytes = electrumX.getRawTransaction(txHash); + + assertFalse(rawTransactionBytes.length == 0); + } + + @Test + public void testGetUnknownRawTransaction() { + ElectrumX electrumX = getInstance(); + + byte[] txHash = HashCode.fromString("f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0").asBytes(); + + try { + electrumX.getRawTransaction(txHash); + fail("Bitcoin transaction should be unknown and hence throw exception"); + } catch (ForeignBlockchainException e) { + if (!(e instanceof ForeignBlockchainException.NotFoundException)) + fail("Bitcoin transaction should be unknown and hence throw NotFoundException"); + } + } + + @Test + public void testGetTransaction() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + String txHash = "7653fea9ffcd829d45ed2672938419a94951b08175982021e77d619b553f29af"; + + BitcoinyTransaction transaction = electrumX.getTransaction(txHash); + + assertNotNull(transaction); + assertTrue(transaction.txHash.equals(txHash)); + } + + @Test + public void testGetUnknownTransaction() { + ElectrumX electrumX = getInstance(); + + String txHash = "f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0"; + + try { + electrumX.getTransaction(txHash); + fail("Bitcoin transaction should be unknown and hence throw exception"); + } catch (ForeignBlockchainException e) { + if (!(e instanceof ForeignBlockchainException.NotFoundException)) + fail("Bitcoin transaction should be unknown and hence throw NotFoundException"); + } + } + + @Test + public void testGetAddressTransactions() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + Address address = Address.fromString(TestNet3Params.get(), "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + + List transactionHashes = electrumX.getAddressTransactions(script, false); + + assertFalse(transactionHashes.isEmpty()); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/HtlcTests.java b/src/test/java/org/qortal/test/crosschain/HtlcTests.java new file mode 100644 index 000000000..75b290bf0 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/HtlcTests.java @@ -0,0 +1,128 @@ +package org.qortal.test.crosschain; + +import static org.junit.Assert.*; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crypto.Crypto; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.repository.DataException; +import org.qortal.test.common.Common; + +import com.google.common.primitives.Longs; + +public class HtlcTests extends Common { + + private Bitcoin bitcoin; + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); // TestNet3 + bitcoin = Bitcoin.getInstance(); + } + + @After + public void afterTest() { + Bitcoin.resetForTesting(); + bitcoin = null; + } + + @Test + public void testFindHtlcSecret() throws ForeignBlockchainException { + // This actually exists on TEST3 but can take a while to fetch + String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + + byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); + byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); + + assertNotNull(secret); + assertArrayEquals("secret incorrect", expectedSecret, secret); + } + + @Test + @Ignore(value = "Doesn't work, to be fixed later") + public void testHtlcSecretCaching() throws ForeignBlockchainException { + String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); + + do { + // We need to perform fresh setup for 1st test + Bitcoin.resetForTesting(); + bitcoin = Bitcoin.getInstance(); + + long now = System.currentTimeMillis(); + long timestampBoundary = now / 30_000L; + + byte[] secret1 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); + long executionPeriod1 = System.currentTimeMillis() - now; + + assertNotNull(secret1); + assertArrayEquals("secret1 incorrect", expectedSecret, secret1); + + assertTrue("1st execution period should not be instant!", executionPeriod1 > 10); + + byte[] secret2 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); + long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1; + + assertNotNull(secret2); + assertArrayEquals("secret2 incorrect", expectedSecret, secret2); + + // Test is only valid if we've called within same timestampBoundary + if (System.currentTimeMillis() / 30_000L != timestampBoundary) + continue; + + assertArrayEquals(secret1, secret2); + + assertTrue("2st execution period should be effectively instant!", executionPeriod2 < 10); + } while (false); + } + + @Test + public void testDetermineHtlcStatus() throws ForeignBlockchainException { + // This actually exists on TEST3 but can take a while to fetch + String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + + BitcoinyHTLC.Status htlcStatus = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L); + assertNotNull(htlcStatus); + + System.out.println(String.format("HTLC %s status: %s", p2shAddress, htlcStatus.name())); + } + + @Test + public void testHtlcStatusCaching() throws ForeignBlockchainException { + do { + // We need to perform fresh setup for 1st test + Bitcoin.resetForTesting(); + bitcoin = Bitcoin.getInstance(); + + long now = System.currentTimeMillis(); + long timestampBoundary = now / 30_000L; + + // Won't ever exist + String p2shAddress = bitcoin.deriveP2shAddress(Crypto.hash160(Longs.toByteArray(now))); + + BitcoinyHTLC.Status htlcStatus1 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L); + long executionPeriod1 = System.currentTimeMillis() - now; + + assertNotNull(htlcStatus1); + assertTrue("1st execution period should not be instant!", executionPeriod1 > 10); + + BitcoinyHTLC.Status htlcStatus2 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L); + long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1; + + assertNotNull(htlcStatus2); + assertEquals(htlcStatus1, htlcStatus2); + + // Test is only valid if we've called within same timestampBoundary + if (System.currentTimeMillis() / 30_000L != timestampBoundary) + continue; + + assertTrue("2st execution period should be effectively instant!", executionPeriod2 < 10); + } while (false); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/LitecoinTests.java b/src/test/java/org/qortal/test/crosschain/LitecoinTests.java new file mode 100644 index 000000000..64837347c --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/LitecoinTests.java @@ -0,0 +1,114 @@ +package org.qortal.test.crosschain; + +import static org.junit.Assert.*; + +import java.util.Arrays; + +import org.bitcoinj.core.Transaction; +import org.bitcoinj.store.BlockStoreException; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.repository.DataException; +import org.qortal.test.common.Common; + +public class LitecoinTests extends Common { + + private Litecoin litecoin; + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); // TestNet3 + litecoin = Litecoin.getInstance(); + } + + @After + public void afterTest() { + Litecoin.resetForTesting(); + litecoin = null; + } + + @Test + public void testGetMedianBlockTime() throws BlockStoreException, ForeignBlockchainException { + long before = System.currentTimeMillis(); + System.out.println(String.format("Bitcoin median blocktime: %d", litecoin.getMedianBlockTime())); + long afterFirst = System.currentTimeMillis(); + + System.out.println(String.format("Bitcoin median blocktime: %d", litecoin.getMedianBlockTime())); + long afterSecond = System.currentTimeMillis(); + + long firstPeriod = afterFirst - before; + long secondPeriod = afterSecond - afterFirst; + + System.out.println(String.format("1st call: %d ms, 2nd call: %d ms", firstPeriod, secondPeriod)); + + assertTrue("2nd call should be quicker than 1st", secondPeriod < firstPeriod); + assertTrue("2nd call should take less than 5 seconds", secondPeriod < 5000L); + } + + @Test + @Ignore(value = "Doesn't work, to be fixed later") + public void testFindHtlcSecret() throws ForeignBlockchainException { + // This actually exists on TEST3 but can take a while to fetch + String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + + byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); + byte[] secret = BitcoinyHTLC.findHtlcSecret(litecoin, p2shAddress); + + assertNotNull("secret not found", secret); + assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret)); + } + + @Test + public void testBuildSpend() { + String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; + + String recipient = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + long amount = 1000L; + + Transaction transaction = litecoin.buildSpend(xprv58, recipient, amount); + assertNotNull("insufficient funds", transaction); + + // Check spent key caching doesn't affect outcome + + transaction = litecoin.buildSpend(xprv58, recipient, amount); + assertNotNull("insufficient funds", transaction); + } + + @Test + public void testGetWalletBalance() { + String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; + + Long balance = litecoin.getWalletBalance(xprv58); + + assertNotNull(balance); + + System.out.println(litecoin.format(balance)); + + // Check spent key caching doesn't affect outcome + + Long repeatBalance = litecoin.getWalletBalance(xprv58); + + assertNotNull(repeatBalance); + + System.out.println(litecoin.format(repeatBalance)); + + assertEquals(balance, repeatBalance); + } + + @Test + public void testGetUnusedReceiveAddress() throws ForeignBlockchainException { + String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; + + String address = litecoin.getUnusedReceiveAddress(xprv58); + + assertNotNull(address); + + System.out.println(address); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/BuildHTLC.java b/src/test/java/org/qortal/test/crosschain/apps/BuildHTLC.java new file mode 100644 index 000000000..fa92fde71 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/BuildHTLC.java @@ -0,0 +1,114 @@ +package org.qortal.test.crosschain.apps; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.BitcoinyHTLC; + +import com.google.common.hash.HashCode; + +public class BuildHTLC { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: BuildHTLC (-b | -l) ")); + System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); + System.err.println(String.format("example: BuildHTLC -l " + + "msAfaDaJ8JiprxxFaAXEEPxKK3JaZCYpLv \\\n" + + "\t0.00008642 \\\n" + + "\tmrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h \\\n" + + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" + + "\t1600000000")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length < 6 || args.length > 6) + usage(null); + + Common.init(); + + Bitcoiny bitcoiny = null; + NetworkParameters params = null; + + Address refundAddress = null; + Coin amount = null; + Address redeemAddress = null; + byte[] hashOfSecret = null; + int lockTime = 0; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + params = bitcoiny.getNetworkParameters(); + + refundAddress = Address.fromString(params, args[argIndex++]); + if (refundAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Refund address must be in P2PKH form"); + + amount = Coin.parseCoin(args[argIndex++]); + + redeemAddress = Address.fromString(params, args[argIndex++]); + if (redeemAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Redeem address must be in P2PKH form"); + + hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); + if (hashOfSecret.length != 20) + usage("Hash of secret must be 20 bytes"); + + lockTime = Integer.parseInt(args[argIndex++]); + int refundTimeoutDelay = lockTime - (int) (System.currentTimeMillis() / 1000L); + if (refundTimeoutDelay < 600 || refundTimeoutDelay > 30 * 24 * 60 * 60) + usage("Locktime (seconds) should be at between 10 minutes and 1 month from now"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny)); + if (p2shFee.isZero()) + return; + + System.out.println(String.format("Refund address: %s", refundAddress)); + System.out.println(String.format("Amount: %s", amount.toPlainString())); + System.out.println(String.format("Redeem address: %s", redeemAddress)); + System.out.println(String.format("Refund/redeem miner's fee: %s", bitcoiny.format(p2shFee))); + System.out.println(String.format("Script lockTime: %s (%d)", LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC), lockTime)); + System.out.println(String.format("Hash of secret: %s", HashCode.fromBytes(hashOfSecret))); + + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(refundAddress.getHash(), lockTime, redeemAddress.getHash(), hashOfSecret); + System.out.println(String.format("Raw script bytes: %s", HashCode.fromBytes(redeemScriptBytes))); + + String p2shAddress = bitcoiny.deriveP2shAddress(redeemScriptBytes); + System.out.println(String.format("P2SH address: %s", p2shAddress)); + + amount = amount.add(p2shFee); + + // Fund P2SH + System.out.println(String.format("\nYou need to fund %s with %s (includes redeem/refund fee of %s)", + p2shAddress, bitcoiny.format(amount), bitcoiny.format(p2shFee))); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/CheckHTLC.java b/src/test/java/org/qortal/test/crosschain/apps/CheckHTLC.java new file mode 100644 index 000000000..8b1cc423d --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/CheckHTLC.java @@ -0,0 +1,135 @@ +package org.qortal.test.crosschain.apps; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.LegacyAddress; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crypto.Crypto; + +import com.google.common.hash.HashCode; + +public class CheckHTLC { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: CheckHTLC (-b | -l) ")); + System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); + System.err.println(String.format("example: CheckP2SH -l " + + "2N4378NbEVGjmiUmoUD9g1vCY6kyx9tDUJ6 \\\n" + + "msAfaDaJ8JiprxxFaAXEEPxKK3JaZCYpLv \\\n" + + "\t0.00008642 \\\n" + + "\tmrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h \\\n" + + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" + + "\t1600184800")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length < 7 || args.length > 7) + usage(null); + + Common.init(); + + Bitcoiny bitcoiny = null; + NetworkParameters params = null; + + Address p2shAddress = null; + Address refundAddress = null; + Coin amount = null; + Address redeemAddress = null; + byte[] hashOfSecret = null; + int lockTime = 0; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + params = bitcoiny.getNetworkParameters(); + + p2shAddress = Address.fromString(params, args[argIndex++]); + if (p2shAddress.getOutputScriptType() != ScriptType.P2SH) + usage("P2SH address invalid"); + + refundAddress = Address.fromString(params, args[argIndex++]); + if (refundAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Refund address must be in P2PKH form"); + + amount = Coin.parseCoin(args[argIndex++]); + + redeemAddress = Address.fromString(params, args[argIndex++]); + if (redeemAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Redeem address must be in P2PKH form"); + + hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); + if (hashOfSecret.length != 20) + usage("Hash of secret must be 20 bytes"); + + lockTime = Integer.parseInt(args[argIndex++]); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny)); + if (p2shFee.isZero()) + return; + + System.out.println(String.format("P2SH address: %s", p2shAddress)); + System.out.println(String.format("Refund PKH: %s", refundAddress)); + System.out.println(String.format("Redeem/refund amount: %s", amount.toPlainString())); + System.out.println(String.format("Redeem PKH: %s", redeemAddress)); + System.out.println(String.format("Hash of secret: %s", HashCode.fromBytes(hashOfSecret))); + System.out.println(String.format("Script lockTime: %s (%d)", LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC), lockTime)); + + System.out.println(String.format("Redeem/refund miner's fee: %s", bitcoiny.format(p2shFee))); + + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(refundAddress.getHash(), lockTime, redeemAddress.getHash(), hashOfSecret); + System.out.println(String.format("Raw script bytes: %s", HashCode.fromBytes(redeemScriptBytes))); + + byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + Address derivedP2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); + + if (!derivedP2shAddress.equals(p2shAddress)) { + System.err.println(String.format("Derived P2SH address %s does not match given address %s", derivedP2shAddress, p2shAddress)); + System.exit(2); + } + + amount = amount.add(p2shFee); + + // Check network's median block time + int medianBlockTime = Common.checkMedianBlockTime(bitcoiny, null); + if (medianBlockTime == 0) + return; + + // Check P2SH is funded + Common.getBalance(bitcoiny, p2shAddress.toString()); + + // Grab all unspent outputs + Common.getUnspentOutputs(bitcoiny, p2shAddress.toString()); + + Common.determineHtlcStatus(bitcoiny, p2shAddress.toString(), amount.value); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/Common.java b/src/test/java/org/qortal/test/crosschain/apps/Common.java new file mode 100644 index 000000000..78066fe7c --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/Common.java @@ -0,0 +1,158 @@ +package org.qortal.test.crosschain.apps; + +import java.security.Security; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Collections; +import java.util.List; + +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.TransactionOutput; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.settings.Settings; +import org.qortal.utils.NTP; + +import com.google.common.hash.HashCode; + +public abstract class Common { + + public static void init() { + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + + Settings.fileInstance("settings-test.json"); + + NTP.setFixedOffset(0L); + } + + public static long getP2shFee(Bitcoiny bitcoiny) { + long p2shFee; + + try { + p2shFee = bitcoiny.getP2shFee(null); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Unable to determine P2SH fee: %s", e.getMessage())); + return 0; + } + + return p2shFee; + } + + public static int checkMedianBlockTime(Bitcoiny bitcoiny, Integer lockTime) { + int medianBlockTime; + + try { + medianBlockTime = bitcoiny.getMedianBlockTime(); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Unable to determine median block time: %s", e.getMessage())); + return 0; + } + + System.out.println(String.format("Median block time: %s", LocalDateTime.ofInstant(Instant.ofEpochSecond(medianBlockTime), ZoneOffset.UTC))); + + long now = System.currentTimeMillis(); + + if (now < medianBlockTime * 1000L) { + System.out.println(String.format("Too soon (%s) based on median block time %s", + LocalDateTime.ofInstant(Instant.ofEpochMilli(now), ZoneOffset.UTC), + LocalDateTime.ofInstant(Instant.ofEpochSecond(medianBlockTime), ZoneOffset.UTC))); + return 0; + } + + if (lockTime != null && now < lockTime * 1000L) { + System.err.println(String.format("Too soon (%s) based on lockTime %s", + LocalDateTime.ofInstant(Instant.ofEpochMilli(now), ZoneOffset.UTC), + LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC))); + return 0; + } + + return medianBlockTime; + } + + public static long getBalance(Bitcoiny bitcoiny, String address58) { + long balance; + + try { + balance = bitcoiny.getConfirmedBalance(address58); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Unable to check address %s balance: %s", address58, e.getMessage())); + return 0; + } + + System.out.println(String.format("Address %s balance: %s", address58, bitcoiny.format(balance))); + + return balance; + } + + public static List getUnspentOutputs(Bitcoiny bitcoiny, String address58) { + List unspentOutputs = Collections.emptyList(); + + try { + unspentOutputs = bitcoiny.getUnspentOutputs(address58); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Can't find unspent outputs for %s: %s", address58, e.getMessage())); + return unspentOutputs; + } + + System.out.println(String.format("Found %d output%s for %s", + unspentOutputs.size(), + (unspentOutputs.size() != 1 ? "s" : ""), + address58)); + + for (TransactionOutput fundingOutput : unspentOutputs) + System.out.println(String.format("Output %s:%d amount %s", + HashCode.fromBytes(fundingOutput.getParentTransactionHash().getBytes()), fundingOutput.getIndex(), + bitcoiny.format(fundingOutput.getValue()))); + + if (unspentOutputs.isEmpty()) + System.err.println(String.format("Can't use spent/unfunded %s", address58)); + + if (unspentOutputs.size() != 1) + System.err.println(String.format("Expecting only one unspent output?")); + + return unspentOutputs; + } + + public static BitcoinyHTLC.Status determineHtlcStatus(Bitcoiny bitcoiny, String address58, long minimumAmount) { + BitcoinyHTLC.Status htlcStatus = null; + + try { + htlcStatus = BitcoinyHTLC.determineHtlcStatus(bitcoiny.getBlockchainProvider(), address58, minimumAmount); + + System.out.println(String.format("HTLC status: %s", htlcStatus.name())); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Unable to determine HTLC status: %s", e.getMessage())); + } + + return htlcStatus; + } + + public static void broadcastTransaction(Bitcoiny bitcoiny, Transaction transaction) { + byte[] rawTransactionBytes = transaction.bitcoinSerialize(); + + System.out.println(String.format("%nRaw transaction bytes:%n%s%n", HashCode.fromBytes(rawTransactionBytes).toString())); + + for (int countDown = 5; countDown >= 1; --countDown) { + System.out.print(String.format("\rBroadcasting transaction in %d second%s... use CTRL-C to abort ", countDown, (countDown != 1 ? "s" : ""))); + try { + Thread.sleep(1000L); + } catch (InterruptedException e) { + System.exit(0); + } + } + System.out.println("Broadcasting transaction... "); + + try { + bitcoiny.broadcastTransaction(transaction); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Failed to broadcast transaction: %s", e.getMessage())); + System.exit(1); + } + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/GetNextReceiveAddress.java b/src/test/java/org/qortal/test/crosschain/apps/GetNextReceiveAddress.java new file mode 100644 index 000000000..ef22355b7 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/GetNextReceiveAddress.java @@ -0,0 +1,78 @@ +package org.qortal.test.crosschain.apps; + +import java.security.Security; + +import org.bitcoinj.core.AddressFormatException; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Litecoin; +import org.qortal.settings.Settings; + +public class GetNextReceiveAddress { + + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + } + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: GetNextReceiveAddress (-b | -l) ")); + System.err.println(String.format("example (testnet): GetNextReceiveAddress -l tpubD6NzVbkrYhZ4X3jV96Wo3Kr8Au2v9cUUEmPRk1smwduFrRVfBjkkw49rRYjgff1fGSktFMfabbvv8b1dmfyLjjbDax6QGyxpsNsx5PXukCB")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 2) + usage(null); + + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + + Settings.fileInstance("settings-test.json"); + + Bitcoiny bitcoiny = null; + String key58 = null; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + + key58 = args[argIndex++]; + + if (!bitcoiny.isValidDeterministicKey(key58)) + usage("Not valid xprv/xpub/tprv/tpub"); + } catch (NumberFormatException | AddressFormatException e) { + usage(String.format("Argument format exception: %s", e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + String receiveAddress = null; + try { + receiveAddress = bitcoiny.getUnusedReceiveAddress(key58); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Failed to determine next receive address: %s", e.getMessage())); + System.exit(1); + } + + System.out.println(String.format("Next receive address: %s", receiveAddress)); + } + +} diff --git a/src/test/java/org/qortal/test/btcacct/GetTransaction.java b/src/test/java/org/qortal/test/crosschain/apps/GetTransaction.java similarity index 55% rename from src/test/java/org/qortal/test/btcacct/GetTransaction.java rename to src/test/java/org/qortal/test/crosschain/apps/GetTransaction.java index 7f42b10bc..9d903a566 100644 --- a/src/test/java/org/qortal/test/btcacct/GetTransaction.java +++ b/src/test/java/org/qortal/test/crosschain/apps/GetTransaction.java @@ -1,4 +1,4 @@ -package org.qortal.test.btcacct; +package org.qortal.test.crosschain.apps; import java.security.Security; import java.util.List; @@ -6,7 +6,11 @@ import org.bitcoinj.core.AddressFormatException; import org.bitcoinj.core.TransactionOutput; import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.qortal.crosschain.BTC; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Litecoin; import org.qortal.settings.Settings; import com.google.common.hash.HashCode; @@ -22,33 +26,52 @@ private static void usage(String error) { if (error != null) System.err.println(error); - System.err.println(String.format("usage: GetTransaction ")); - System.err.println(String.format("example (mainnet): GetTransaction 816407e79568f165f13e09e9912c5f2243e0a23a007cec425acedc2e89284660")); - System.err.println(String.format("example (testnet): GetTransaction 3bfd17a492a4e3d6cb7204e17e20aca6c1ab82e1828bd1106eefbaf086fb8a4e")); + System.err.println(String.format("usage: GetTransaction (-b | -l) ")); + System.err.println(String.format("example (mainnet): GetTransaction -b 816407e79568f165f13e09e9912c5f2243e0a23a007cec425acedc2e89284660")); + System.err.println(String.format("example (testnet): GetTransaction -b 3bfd17a492a4e3d6cb7204e17e20aca6c1ab82e1828bd1106eefbaf086fb8a4e")); System.exit(1); } public static void main(String[] args) { - if (args.length != 1) + if (args.length != 2) usage(null); Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + Settings.fileInstance("settings-test.json"); + Bitcoiny bitcoiny = null; byte[] transactionId = null; + int argIndex = 0; try { - int argIndex = 0; + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } transactionId = HashCode.fromString(args[argIndex++]).asBytes(); } catch (NumberFormatException | AddressFormatException e) { usage(String.format("Argument format exception: %s", e.getMessage())); } + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + // Grab all outputs from transaction - List fundingOutputs = BTC.getInstance().getOutputs(transactionId); - if (fundingOutputs == null) { - System.out.println(String.format("Transaction not found")); + List fundingOutputs; + try { + fundingOutputs = bitcoiny.getOutputs(transactionId); + } catch (ForeignBlockchainException e) { + System.out.println(String.format("Transaction not found (or error occurred)")); return; } diff --git a/src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java b/src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java new file mode 100644 index 000000000..7a880b1a0 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java @@ -0,0 +1,82 @@ +package org.qortal.test.crosschain.apps; + +import java.security.Security; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +import org.bitcoinj.core.AddressFormatException; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.qortal.crosschain.*; +import org.qortal.settings.Settings; + +public class GetWalletTransactions { + + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + } + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: GetWalletTransactions (-b | -l) ")); + System.err.println(String.format("example (testnet): GetWalletTransactions -l tpubD6NzVbkrYhZ4X3jV96Wo3Kr8Au2v9cUUEmPRk1smwduFrRVfBjkkw49rRYjgff1fGSktFMfabbvv8b1dmfyLjjbDax6QGyxpsNsx5PXukCB")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 2) + usage(null); + + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + + Settings.fileInstance("settings-test.json"); + + Bitcoiny bitcoiny = null; + String key58 = null; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + + key58 = args[argIndex++]; + + if (!bitcoiny.isValidDeterministicKey(key58)) + usage("Not valid xprv/xpub/tprv/tpub"); + } catch (NumberFormatException | AddressFormatException e) { + usage(String.format("Argument format exception: %s", e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + // Grab all outputs from transaction + List transactions = null; + try { + transactions = bitcoiny.getWalletTransactions(key58); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Failed to obtain wallet transactions: %s", e.getMessage())); + System.exit(1); + } + + System.out.println(String.format("Found %d transaction%s", transactions.size(), (transactions.size() != 1 ? "s" : ""))); + + for (SimpleTransaction transaction : transactions.stream().sorted(Comparator.comparingInt(SimpleTransaction::getTimestamp)).collect(Collectors.toList())) + System.out.println(String.format("%s", transaction)); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/Pay.java b/src/test/java/org/qortal/test/crosschain/apps/Pay.java new file mode 100644 index 000000000..93c7aede7 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/Pay.java @@ -0,0 +1,80 @@ +package org.qortal.test.crosschain.apps; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.core.Transaction; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.Litecoin; + +public class Pay { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: Pay (-b | -l) ")); + System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); + System.err.println(String.format("example: Pay -l " + + "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ \\\n" + + "\tmsAfaDaJ8JiprxxFaAXEEPxKK3JaZCYpLv \\\n" + + "\t0.00008642")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length < 4 || args.length > 4) + usage(null); + + Common.init(); + + Bitcoiny bitcoiny = null; + NetworkParameters params = null; + + String xprv58 = null; + Address address = null; + Coin amount = null; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + params = bitcoiny.getNetworkParameters(); + + xprv58 = args[argIndex++]; + if (!bitcoiny.isValidDeterministicKey(xprv58)) + usage("xprv invalid"); + + address = Address.fromString(params, args[argIndex++]); + + amount = Coin.parseCoin(args[argIndex++]); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + System.out.println(String.format("Address: %s", address)); + System.out.println(String.format("Amount: %s", amount.toPlainString())); + + Transaction transaction = bitcoiny.buildSpend(xprv58, address.toString(), amount.value); + if (transaction == null) { + System.err.println("Insufficent funds"); + System.exit(1); + } + + Common.broadcastTransaction(bitcoiny, transaction); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/RedeemHTLC.java b/src/test/java/org/qortal/test/crosschain/apps/RedeemHTLC.java new file mode 100644 index 000000000..d4f1bcf17 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/RedeemHTLC.java @@ -0,0 +1,166 @@ +package org.qortal.test.crosschain.apps; + +import java.util.Arrays; +import java.util.List; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.ECKey; +import org.bitcoinj.core.LegacyAddress; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.TransactionOutput; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crypto.Crypto; + +import com.google.common.hash.HashCode; + +public class RedeemHTLC { + + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + } + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: Redeem (-b | -l) ")); + System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); + System.err.println(String.format("example: Redeem -l " + + "2N4378NbEVGjmiUmoUD9g1vCY6kyx9tDUJ6 \\\n" + + "\tmsAfaDaJ8JiprxxFaAXEEPxKK3JaZCYpLv \\\n" + + "\tefdaed23c4bc85c8ccae40d774af3c2a10391c648b6420cdd83cd44c27fcb5955201c64e372d \\\n" + + "\t5468697320737472696e672069732065786163746c7920333220627974657321 \\\n" + + "\t1600184800 \\\n" + + "\tmrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length < 7 || args.length > 7) + usage(null); + + Common.init(); + + Bitcoiny bitcoiny = null; + NetworkParameters params = null; + + Address p2shAddress = null; + Address refundAddress = null; + byte[] redeemPrivateKey = null; + byte[] secret = null; + int lockTime = 0; + Address outputAddress = null; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + params = bitcoiny.getNetworkParameters(); + + p2shAddress = Address.fromString(params, args[argIndex++]); + if (p2shAddress.getOutputScriptType() != ScriptType.P2SH) + usage("P2SH address invalid"); + + refundAddress = Address.fromString(params, args[argIndex++]); + if (refundAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Refund address must be in P2PKH form"); + + redeemPrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); + // Auto-trim + if (redeemPrivateKey.length >= 37 && redeemPrivateKey.length <= 38) + redeemPrivateKey = Arrays.copyOfRange(redeemPrivateKey, 1, 33); + if (redeemPrivateKey.length != 32) + usage("Redeem private key must be 32 bytes"); + + secret = HashCode.fromString(args[argIndex++]).asBytes(); + if (secret.length == 0) + usage("Invalid secret bytes"); + + lockTime = Integer.parseInt(args[argIndex++]); + + outputAddress = Address.fromString(params, args[argIndex++]); + if (outputAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Output address invalid"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny)); + if (p2shFee.isZero()) + return; + + System.out.println(String.format("Attempting to redeem HTLC %s to %s", p2shAddress, outputAddress)); + + byte[] hashOfSecret = Crypto.hash160(secret); + + ECKey redeemKey = ECKey.fromPrivate(redeemPrivateKey); + Address redeemAddress = Address.fromKey(params, redeemKey, ScriptType.P2PKH); + + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(refundAddress.getHash(), lockTime, redeemAddress.getHash(), hashOfSecret); + + byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + Address derivedP2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); + + if (!derivedP2shAddress.equals(p2shAddress)) { + System.err.println(String.format("Raw script bytes: %s", HashCode.fromBytes(redeemScriptBytes))); + System.err.println(String.format("Derived P2SH address %s does not match given address %s", derivedP2shAddress, p2shAddress)); + System.exit(2); + return; + } + + // Actual live processing... + + int medianBlockTime = Common.checkMedianBlockTime(bitcoiny, null); + if (medianBlockTime == 0) + return; + + // Check P2SH is funded + long p2shBalance = Common.getBalance(bitcoiny, p2shAddress.toString()); + if (p2shBalance == 0) + return; + + // Grab all unspent outputs + List unspentOutputs = Common.getUnspentOutputs(bitcoiny, p2shAddress.toString()); + if (unspentOutputs.isEmpty()) + return; + + Coin redeemAmount = Coin.valueOf(p2shBalance).subtract(p2shFee); + + BitcoinyHTLC.Status htlcStatus = Common.determineHtlcStatus(bitcoiny, p2shAddress.toString(), redeemAmount.value); + if (htlcStatus == null) + return; + + if (htlcStatus != BitcoinyHTLC.Status.FUNDED) { + System.err.println(String.format("Expecting %s HTLC status, but actual status is %s", "FUNDED", htlcStatus.name())); + System.exit(2); + return; + } + + System.out.println(String.format("Spending %s of outputs, with %s as mining fee", bitcoiny.format(redeemAmount), bitcoiny.format(p2shFee))); + + Transaction redeemTransaction = BitcoinyHTLC.buildRedeemTransaction(bitcoiny.getNetworkParameters(), redeemAmount, redeemKey, + unspentOutputs, redeemScriptBytes, secret, outputAddress.getHash()); + + Common.broadcastTransaction(bitcoiny, redeemTransaction); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/RefundHTLC.java b/src/test/java/org/qortal/test/crosschain/apps/RefundHTLC.java new file mode 100644 index 000000000..723185f0f --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/RefundHTLC.java @@ -0,0 +1,163 @@ +package org.qortal.test.crosschain.apps; + +import java.util.Arrays; +import java.util.List; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.ECKey; +import org.bitcoinj.core.LegacyAddress; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.TransactionOutput; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crypto.Crypto; + +import com.google.common.hash.HashCode; + +public class RefundHTLC { + + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + } + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: RefundHTLC (-b | -l) ")); + System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); + System.err.println(String.format("example: RefundHTLC -l " + + "2N4378NbEVGjmiUmoUD9g1vCY6kyx9tDUJ6 \\\n" + + "\tef8f31b49c31b4a140aebcd9605fded88cc2dad0844c4b984f9191a5a416f72d3801e16447b0 \\\n" + + "\tmrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h \\\n" + + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" + + "\t1600184800 \\\n" + + "\tmoJtbbhs7T4Z5hmBH2iyKhGrCWBzQWS2CL")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length < 7 || args.length > 7) + usage(null); + + Common.init(); + + Bitcoiny bitcoiny = null; + NetworkParameters params = null; + + Address p2shAddress = null; + byte[] refundPrivateKey = null; + Address redeemAddress = null; + byte[] hashOfSecret = null; + int lockTime = 0; + Address outputAddress = null; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + params = bitcoiny.getNetworkParameters(); + + p2shAddress = Address.fromString(params, args[argIndex++]); + if (p2shAddress.getOutputScriptType() != ScriptType.P2SH) + usage("P2SH address invalid"); + + refundPrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); + // Auto-trim + if (refundPrivateKey.length >= 37 && refundPrivateKey.length <= 38) + refundPrivateKey = Arrays.copyOfRange(refundPrivateKey, 1, 33); + if (refundPrivateKey.length != 32) + usage("Refund private key must be 32 bytes"); + + redeemAddress = Address.fromString(params, args[argIndex++]); + if (redeemAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Redeem address must be in P2PKH form"); + + hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); + if (hashOfSecret.length != 20) + usage("HASH160 of secret must be 20 bytes"); + + lockTime = Integer.parseInt(args[argIndex++]); + + outputAddress = Address.fromString(params, args[argIndex++]); + if (outputAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Output address invalid"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny)); + if (p2shFee.isZero()) + return; + + System.out.println(String.format("Attempting to refund HTLC %s to %s", p2shAddress, outputAddress)); + + ECKey refundKey = ECKey.fromPrivate(refundPrivateKey); + Address refundAddress = Address.fromKey(params, refundKey, ScriptType.P2PKH); + + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(refundAddress.getHash(), lockTime, redeemAddress.getHash(), hashOfSecret); + + byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + Address derivedP2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); + + if (!derivedP2shAddress.equals(p2shAddress)) { + System.err.println(String.format("Raw script bytes: %s", HashCode.fromBytes(redeemScriptBytes))); + System.err.println(String.format("Derived P2SH address %s does not match given address %s", derivedP2shAddress, p2shAddress)); + System.exit(2); + } + + // Actual live processing... + + int medianBlockTime = Common.checkMedianBlockTime(bitcoiny, lockTime); + if (medianBlockTime == 0) + return; + + // Check P2SH is funded + long p2shBalance = Common.getBalance(bitcoiny, p2shAddress.toString()); + if (p2shBalance == 0) + return; + + // Grab all unspent outputs + List unspentOutputs = Common.getUnspentOutputs(bitcoiny, p2shAddress.toString()); + if (unspentOutputs.isEmpty()) + return; + + Coin refundAmount = Coin.valueOf(p2shBalance).subtract(p2shFee); + + BitcoinyHTLC.Status htlcStatus = Common.determineHtlcStatus(bitcoiny, p2shAddress.toString(), refundAmount.value); + if (htlcStatus == null) + return; + + if (htlcStatus != BitcoinyHTLC.Status.FUNDED) { + System.err.println(String.format("Expecting %s HTLC status, but actual status is %s", "FUNDED", htlcStatus.name())); + System.exit(2); + return; + } + + System.out.println(String.format("Spending %s of outputs, with %s as mining fee", bitcoiny.format(refundAmount), bitcoiny.format(p2shFee))); + + Transaction refundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoiny.getNetworkParameters(), refundAmount, refundKey, + unspentOutputs, redeemScriptBytes, lockTime, outputAddress.getHash()); + + Common.broadcastTransaction(bitcoiny, refundTransaction); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/bitcoinv1/BitcoinACCTv1Tests.java b/src/test/java/org/qortal/test/crosschain/bitcoinv1/BitcoinACCTv1Tests.java new file mode 100644 index 000000000..4487e8744 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/bitcoinv1/BitcoinACCTv1Tests.java @@ -0,0 +1,795 @@ +package org.qortal.test.crosschain.bitcoinv1; + +import static org.junit.Assert.*; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.function.Function; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.Account; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.block.Block; +import org.qortal.crosschain.BitcoinACCTv1; +import org.qortal.crosschain.AcctMode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.utils.Amounts; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; + +public class BitcoinACCTv1Tests extends Common { + + public static final byte[] secretA = "This string is exactly 32 bytes!".getBytes(); + public static final byte[] hashOfSecretA = Crypto.hash160(secretA); // daf59884b4d1aec8c1b17102530909ee43c0151a + public static final byte[] secretB = "This string is roughly 32 bytes?".getBytes(); + public static final byte[] hashOfSecretB = Crypto.hash160(secretB); // 31f0dd71decf59bbc8ef0661f4030479255cfa58 + public static final byte[] bitcoinPublicKeyHash = HashCode.fromString("bb00bb11bb22bb33bb44bb55bb66bb77bb88bb99").asBytes(); + public static final int tradeTimeout = 20; // blocks + public static final long redeemAmount = 80_40200000L; + public static final long fundingAmount = 123_45600000L; + public static final long bitcoinAmount = 864200L; // 0.00864200 BTC + + private static final Random RANDOM = new Random(); + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testCompile() { + PrivateKeyAccount tradeAccount = createTradeAccount(null); + + byte[] creationBytes = BitcoinACCTv1.buildQortalAT(tradeAccount.getAddress(), bitcoinPublicKeyHash, hashOfSecretB, redeemAmount, bitcoinAmount, tradeTimeout); + assertNotNull(creationBytes); + + System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); + } + + @Test + public void testDeploy() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + + long expectedBalance = deployersInitialBalance - fundingAmount - deployAtTransaction.getTransactionData().getFee(); + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = fundingAmount; + actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); + + assertEquals("AT's post-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = partnersInitialBalance; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-deployment balance incorrect", expectedBalance, actualBalance); + + // Test orphaning + BlockUtils.orphanLastBlock(repository); + + expectedBalance = deployersInitialBalance; + actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = 0; + actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); + + assertEquals("AT's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = partnersInitialBalance; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testOfferCancel() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + // Send creator's address to AT, instead of typical partner's address + byte[] messageData = BitcoinACCTv1.getInstance().buildCancelMessage(deployer.getAddress()); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); + long messageFee = messageTransaction.getTransactionData().getFee(); + + // AT should process 'cancel' message in next block + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + + // Check balances + long expectedMinimumBalance = deployersPostDeploymentBalance; + long expectedMaximumBalance = deployersInitialBalance - deployAtFee - messageFee; + + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); + assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); + + // Test orphaning + BlockUtils.orphanLastBlock(repository); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance - messageFee; + actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testOfferCancelInvalidLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + // Instead of sending creator's address to AT, send too-short/invalid message + byte[] messageData = new byte[7]; + RANDOM.nextBytes(messageData); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); + long messageFee = messageTransaction.getTransactionData().getFee(); + + // AT should process 'cancel' message in next block + // As message is too short, it will be padded to 32bytes but cancel code doesn't care about message content, so should be ok + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testTradingInfoProcessing() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + Block postDeploymentBlock = BlockUtils.mintBlock(repository); + int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + describeAt(repository, atAddress); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + + // AT should be in TRADE mode + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check hashOfSecretA was extracted correctly + assertTrue(Arrays.equals(hashOfSecretA, tradeData.hashOfSecretA)); + + // Check trade partner Qortal address was extracted correctly + assertEquals(partner.getAddress(), tradeData.qortalPartnerAddress); + + // Check trade partner's Bitcoin PKH was extracted correctly + assertTrue(Arrays.equals(bitcoinPublicKeyHash, tradeData.partnerForeignPKH)); + + // Test orphaning + BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance; + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + // TEST SENDING TRADING INFO BUT NOT FROM AT CREATOR (SHOULD BE IGNORED) + @SuppressWarnings("unused") + @Test + public void testIncorrectTradeSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT BUT NOT FROM AT CREATOR + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, bystander, messageData, atAddress); + + BlockUtils.mintBlock(repository); + + long expectedBalance = partnersInitialBalance; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-initial-payout balance incorrect", expectedBalance, actualBalance); + + describeAt(repository, atAddress); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + + // AT should still be in OFFER mode + assertEquals(AcctMode.OFFERING, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testAutomaticTradeRefund() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + Block postDeploymentBlock = BlockUtils.mintBlock(repository); + int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); + + // Check refund + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in REFUNDED mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.REFUNDED, tradeData.mode); + + // Test orphaning + BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance; + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretsCorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secrets to AT, from correct account + messageData = BitcoinACCTv1.buildRedeemMessage(secretA, secretB, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in REDEEMED mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.REDEEMED, tradeData.mode); + + // Check balances + long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee() + redeemAmount; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-redeem balance incorrect", expectedBalance, actualBalance); + + // Orphan redeem + BlockUtils.orphanLastBlock(repository); + + // Check balances + expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-orphan/pre-redeem balance incorrect", expectedBalance, actualBalance); + + // Check AT state + ATStateData postOrphanAtStateData = repository.getATRepository().getLatestATState(atAddress); + + assertTrue("AT states mismatch", Arrays.equals(preRedeemAtStateData.getStateData(), postOrphanAtStateData.getStateData())); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretsIncorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secrets to AT, but from wrong account + messageData = BitcoinACCTv1.buildRedeemMessage(secretA, secretB, partner.getAddress()); + messageTransaction = sendMessage(repository, bystander, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check balances + long expectedBalance = partnersInitialBalance; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Check eventual refund + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + } + } + + @SuppressWarnings("unused") + @Test + public void testIncorrectSecretsCorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send incorrect secrets to AT, from correct account + byte[] wrongSecret = new byte[32]; + RANDOM.nextBytes(wrongSecret); + messageData = BitcoinACCTv1.buildRedeemMessage(wrongSecret, secretB, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Send incorrect secrets to AT, from correct account + messageData = BitcoinACCTv1.buildRedeemMessage(secretA, wrongSecret, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check balances + expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee() * 2; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Check eventual refund + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretsCorrectSenderInvalidMessageLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secrets to AT, from correct account, but missing receive address, hence incorrect length + messageData = Bytes.concat(secretA, secretB); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should be in TRADING mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testDescribeDeployed() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + + List executableAts = repository.getATRepository().getAllExecutableATs(); + + for (ATData atData : executableAts) { + String atAddress = atData.getATAddress(); + byte[] codeBytes = atData.getCodeBytes(); + byte[] codeHash = Crypto.digest(codeBytes); + + System.out.println(String.format("%s: code length: %d byte%s, code hash: %s", + atAddress, + codeBytes.length, + (codeBytes.length != 1 ? "s": ""), + HashCode.fromBytes(codeHash))); + + // Not one of ours? + if (!Arrays.equals(codeHash, BitcoinACCTv1.CODE_BYTES_HASH)) + continue; + + describeAt(repository, atAddress); + } + } + } + + private int calcTestLockTimeA(long messageTimestamp) { + return (int) (messageTimestamp / 1000L + tradeTimeout * 60); + } + + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, String tradeAddress) throws DataException { + byte[] creationBytes = BitcoinACCTv1.buildQortalAT(tradeAddress, bitcoinPublicKeyHash, hashOfSecretB, redeemAmount, bitcoinAmount, tradeTimeout); + + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "QORT-BTC cross-chain trade"; + String description = String.format("Qortal-Bitcoin cross-chain trade"); + String atType = "ACCT"; + String tags = "QORT-BTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, messageTransactionData, sender); + + return messageTransaction; + } + + private void checkTradeRefund(Repository repository, Account deployer, long deployersInitialBalance, long deployAtFee) throws DataException { + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + int refundTimeout = tradeTimeout * 3 / 4 + 1; // close enough + + // AT should automatically refund deployer after 'refundTimeout' blocks + for (int blockCount = 0; blockCount <= refundTimeout; ++blockCount) + BlockUtils.mintBlock(repository); + + // We don't bother to exactly calculate QORT spent running AT for several blocks, but we do know the expected range + long expectedMinimumBalance = deployersPostDeploymentBalance; + long expectedMaximumBalance = deployersInitialBalance - deployAtFee; + + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); + assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); + } + + private void describeAt(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + + Function epochMilliFormatter = (timestamp) -> LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneOffset.UTC).format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM)); + int currentBlockHeight = repository.getBlockRepository().getBlockchainHeight(); + + System.out.print(String.format("%s:\n" + + "\tmode: %s\n" + + "\tcreator: %s,\n" + + "\tcreation timestamp: %s,\n" + + "\tcurrent balance: %s QORT,\n" + + "\tis finished: %b,\n" + + "\tHASH160 of secret-B: %s,\n" + + "\tredeem payout: %s QORT,\n" + + "\texpected bitcoin: %s BTC,\n" + + "\tcurrent block height: %d,\n", + tradeData.qortalAtAddress, + tradeData.mode, + tradeData.qortalCreator, + epochMilliFormatter.apply(tradeData.creationTimestamp), + Amounts.prettyAmount(tradeData.qortBalance), + atData.getIsFinished(), + HashCode.fromBytes(tradeData.hashOfSecretB).toString().substring(0, 40), + Amounts.prettyAmount(tradeData.qortAmount), + Amounts.prettyAmount(tradeData.expectedForeignAmount), + currentBlockHeight)); + + if (tradeData.mode != AcctMode.OFFERING && tradeData.mode != AcctMode.CANCELLED) { + System.out.println(String.format("\trefund height: block %d,\n" + + "\tHASH160 of secret-A: %s,\n" + + "\tBitcoin P2SH-A nLockTime: %d (%s),\n" + + "\tBitcoin P2SH-B nLockTime: %d (%s),\n" + + "\ttrade partner: %s", + tradeData.tradeRefundHeight, + HashCode.fromBytes(tradeData.hashOfSecretA).toString().substring(0, 40), + tradeData.lockTimeA, epochMilliFormatter.apply(tradeData.lockTimeA * 1000L), + tradeData.lockTimeB, epochMilliFormatter.apply(tradeData.lockTimeB * 1000L), + tradeData.qortalPartnerAddress)); + } + } + + private PrivateKeyAccount createTradeAccount(Repository repository) { + // We actually use a known test account with funds to avoid PoW compute + return Common.getTestAccount(repository, "alice"); + } + +} diff --git a/src/test/java/org/qortal/test/btcacct/DeployAT.java b/src/test/java/org/qortal/test/crosschain/bitcoinv1/DeployAT.java similarity index 66% rename from src/test/java/org/qortal/test/btcacct/DeployAT.java rename to src/test/java/org/qortal/test/crosschain/bitcoinv1/DeployAT.java index 986721645..f27f7a7b0 100644 --- a/src/test/java/org/qortal/test/btcacct/DeployAT.java +++ b/src/test/java/org/qortal/test/crosschain/bitcoinv1/DeployAT.java @@ -1,12 +1,15 @@ -package org.qortal.test.btcacct; +package org.qortal.test.crosschain.bitcoinv1; -import java.security.Security; - -import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bitcoinj.core.Address; +import org.bitcoinj.core.AddressFormatException; +import org.bitcoinj.core.LegacyAddress; +import org.bitcoinj.core.NetworkParameters; import org.qortal.account.PrivateKeyAccount; import org.qortal.asset.Asset; import org.qortal.controller.Controller; -import org.qortal.crosschain.BTCACCT; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.BitcoinACCTv1; +import org.qortal.crosschain.Bitcoiny; import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.DeployAtTransactionData; import org.qortal.data.transaction.TransactionData; @@ -16,7 +19,7 @@ import org.qortal.repository.RepositoryFactory; import org.qortal.repository.RepositoryManager; import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; -import org.qortal.settings.Settings; +import org.qortal.test.crosschain.apps.Common; import org.qortal.transaction.DeployAtTransaction; import org.qortal.transaction.Transaction; import org.qortal.transform.TransformationException; @@ -28,37 +31,37 @@ public class DeployAT { - public static final long atFundingExtra = 2000000L; - private static void usage(String error) { if (error != null) System.err.println(error); - System.err.println(String.format("usage: DeployAT ")); + System.err.println(String.format("usage: DeployAT ")); System.err.println(String.format("example: DeployAT " - + "AdTd9SUEYSdTW8mgK3Gu72K97bCHGdUwi2VvLNjUohot \\\n" - + "\t80.4020 \\\n" + + "7Eztjz2TsxwbrWUYEaSdLbASKQGTfK2rR7ViFc5gaiZw \\\n" + + "\t10 \\\n" + + "\t10.1 \\\n" + "\t0.00864200 \\\n" + + "\t750b06757a2448b8a4abebaa6e4662833fd5ddbb (or mrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h) \\\n" + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" - + "\t0.0001 \\\n" - + "\t123.456 \\\n" - + "\t10")); + + "\t10080")); System.exit(1); } public static void main(String[] args) { - if (args.length != 8) + if (args.length != 7) usage(null); - Security.insertProviderAt(new BouncyCastleProvider(), 0); - Settings.fileInstance("settings-test.json"); + Common.init(); + + Bitcoiny bitcoiny = Bitcoin.getInstance(); + NetworkParameters params = bitcoiny.getNetworkParameters(); byte[] refundPrivateKey = null; long redeemAmount = 0; - long expectedBitcoin = 0; - byte[] secretHash = null; - long initialPayout = 0; long fundingAmount = 0; + long expectedBitcoin = 0; + byte[] bitcoinPublicKeyHash = null; + byte[] hashOfSecret = null; int tradeTimeout = 0; int argIndex = 0; @@ -71,25 +74,33 @@ public static void main(String[] args) { if (redeemAmount <= 0) usage("QORT amount must be positive"); + fundingAmount = Long.parseLong(args[argIndex++]); + if (fundingAmount <= redeemAmount) + usage("AT funding amount must be greater than QORT redeem amount"); + expectedBitcoin = Long.parseLong(args[argIndex++]); if (expectedBitcoin <= 0) usage("Expected BTC amount must be positive"); - secretHash = HashCode.fromString(args[argIndex++]).asBytes(); - if (secretHash.length != 20) - usage("Hash of secret must be 20 bytes"); - - initialPayout = Long.parseLong(args[argIndex++]); - if (initialPayout < 0) - usage("Initial QORT payout must be positive"); + String bitcoinPKHish = args[argIndex++]; + // Try P2PKH first + try { + Address bitcoinAddress = LegacyAddress.fromBase58(params, bitcoinPKHish); + bitcoinPublicKeyHash = bitcoinAddress.getHash(); + } catch (AddressFormatException e) { + // Try parsing as PKH hex string instead + bitcoinPublicKeyHash = HashCode.fromString(bitcoinPKHish).asBytes(); + } + if (bitcoinPublicKeyHash.length != 20) + usage("Bitcoin PKH must be 20 bytes"); - fundingAmount = Long.parseLong(args[argIndex++]); - if (fundingAmount <= redeemAmount) - usage("AT funding amount must be greater than QORT redeem amount"); + hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); + if (hashOfSecret.length != 20) + usage("Hash of secret must be 20 bytes"); tradeTimeout = Integer.parseInt(args[argIndex++]); - if (tradeTimeout < 10 || tradeTimeout > 50000) - usage("AT trade timeout should be between 10 and 50,000 minutes"); + if (tradeTimeout < 60 || tradeTimeout > 50000) + usage("Trade timeout (minutes) must be between 60 and 50000"); } catch (IllegalArgumentException e) { usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); } @@ -98,12 +109,11 @@ public static void main(String[] args) { RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); RepositoryManager.setRepositoryFactory(repositoryFactory); } catch (DataException e) { - throw new RuntimeException("Repository startup issue: " + e.getMessage()); + System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); + System.exit(2); } try (final Repository repository = RepositoryManager.getRepository()) { - System.out.println("Confirm the following is correct based on the info you've given:"); - PrivateKeyAccount refundAccount = new PrivateKeyAccount(repository, refundPrivateKey); System.out.println(String.format("Refund Qortal address: %s", refundAccount.getAddress())); @@ -111,11 +121,11 @@ public static void main(String[] args) { System.out.println(String.format("AT funding amount: %s", Amounts.prettyAmount(fundingAmount))); - System.out.println(String.format("HASH160 of secret: %s", HashCode.fromBytes(secretHash))); + System.out.println(String.format("HASH160 of secret: %s", HashCode.fromBytes(hashOfSecret))); // Deploy AT - byte[] creationBytes = BTCACCT.buildQortalAT(refundAccount.getAddress(), secretHash, tradeTimeout, initialPayout, redeemAmount, expectedBitcoin); - System.out.println("CIYAM AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); + byte[] creationBytes = BitcoinACCTv1.buildQortalAT(refundAccount.getAddress(), bitcoinPublicKeyHash, hashOfSecret, redeemAmount, expectedBitcoin, tradeTimeout); + System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); long txTimestamp = System.currentTimeMillis(); byte[] lastReference = refundAccount.getLastReference(); @@ -149,11 +159,10 @@ public static void main(String[] args) { System.exit(2); } - System.out.println(String.format("\nSigned transaction in base58, ready for POST /transactions/process:\n%s\n", Base58.encode(signedBytes))); - } catch (NumberFormatException e) { - usage(String.format("Number format exception: %s", e.getMessage())); + System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); } catch (DataException e) { - throw new RuntimeException("Repository issue: " + e.getMessage()); + System.err.println(String.format("Repository issue: %s", e.getMessage())); + System.exit(2); } } diff --git a/src/test/java/org/qortal/test/crosschain/dogecoinv3/DogecoinACCTv3Tests.java b/src/test/java/org/qortal/test/crosschain/dogecoinv3/DogecoinACCTv3Tests.java new file mode 100644 index 000000000..7056e4339 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/dogecoinv3/DogecoinACCTv3Tests.java @@ -0,0 +1,769 @@ +package org.qortal.test.crosschain.dogecoinv3; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.Account; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.block.Block; +import org.qortal.crosschain.AcctMode; +import org.qortal.crosschain.DogecoinACCTv3; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.utils.Amounts; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.function.Function; + +import static org.junit.Assert.*; + +public class DogecoinACCTv3Tests extends Common { + + public static final byte[] secretA = "This string is exactly 32 bytes!".getBytes(); + public static final byte[] hashOfSecretA = Crypto.hash160(secretA); // daf59884b4d1aec8c1b17102530909ee43c0151a + public static final byte[] dogecoinPublicKeyHash = HashCode.fromString("bb00bb11bb22bb33bb44bb55bb66bb77bb88bb99").asBytes(); + public static final int tradeTimeout = 20; // blocks + public static final long redeemAmount = 80_40200000L; + public static final long fundingAmount = 123_45600000L; + public static final long dogecoinAmount = 864200L; // 0.00864200 DOGE + + private static final Random RANDOM = new Random(); + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testCompile() { + PrivateKeyAccount tradeAccount = createTradeAccount(null); + + byte[] creationBytes = DogecoinACCTv3.buildQortalAT(tradeAccount.getAddress(), dogecoinPublicKeyHash, redeemAmount, dogecoinAmount, tradeTimeout); + assertNotNull(creationBytes); + + System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); + } + + @Test + public void testDeploy() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + + long expectedBalance = deployersInitialBalance - fundingAmount - deployAtTransaction.getTransactionData().getFee(); + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = fundingAmount; + actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); + + assertEquals("AT's post-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = partnersInitialBalance; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-deployment balance incorrect", expectedBalance, actualBalance); + + // Test orphaning + BlockUtils.orphanLastBlock(repository); + + expectedBalance = deployersInitialBalance; + actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = 0; + actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); + + assertEquals("AT's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = partnersInitialBalance; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testOfferCancel() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + // Send creator's address to AT, instead of typical partner's address + byte[] messageData = DogecoinACCTv3.getInstance().buildCancelMessage(deployer.getAddress()); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); + long messageFee = messageTransaction.getTransactionData().getFee(); + + // AT should process 'cancel' message in next block + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = DogecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + + // Check balances + long expectedMinimumBalance = deployersPostDeploymentBalance; + long expectedMaximumBalance = deployersInitialBalance - deployAtFee - messageFee; + + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); + assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); + + // Test orphaning + BlockUtils.orphanLastBlock(repository); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance - messageFee; + actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testOfferCancelInvalidLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + // Instead of sending creator's address to AT, send too-short/invalid message + byte[] messageData = new byte[7]; + RANDOM.nextBytes(messageData); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); + long messageFee = messageTransaction.getTransactionData().getFee(); + + // AT should process 'cancel' message in next block + // As message is too short, it will be padded to 32bytes but cancel code doesn't care about message content, so should be ok + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = DogecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testTradingInfoProcessing() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = DogecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = DogecoinACCTv3.buildTradeMessage(partner.getAddress(), dogecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + Block postDeploymentBlock = BlockUtils.mintBlock(repository); + int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + describeAt(repository, atAddress); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = DogecoinACCTv3.getInstance().populateTradeData(repository, atData); + + // AT should be in TRADE mode + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check hashOfSecretA was extracted correctly + assertTrue(Arrays.equals(hashOfSecretA, tradeData.hashOfSecretA)); + + // Check trade partner Qortal address was extracted correctly + assertEquals(partner.getAddress(), tradeData.qortalPartnerAddress); + + // Check trade partner's dogecoin PKH was extracted correctly + assertTrue(Arrays.equals(dogecoinPublicKeyHash, tradeData.partnerForeignPKH)); + + // Test orphaning + BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance; + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + // TEST SENDING TRADING INFO BUT NOT FROM AT CREATOR (SHOULD BE IGNORED) + @SuppressWarnings("unused") + @Test + public void testIncorrectTradeSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = DogecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT BUT NOT FROM AT CREATOR + byte[] messageData = DogecoinACCTv3.buildTradeMessage(partner.getAddress(), dogecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, bystander, messageData, atAddress); + + BlockUtils.mintBlock(repository); + + long expectedBalance = partnersInitialBalance; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-initial-payout balance incorrect", expectedBalance, actualBalance); + + describeAt(repository, atAddress); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = DogecoinACCTv3.getInstance().populateTradeData(repository, atData); + + // AT should still be in OFFER mode + assertEquals(AcctMode.OFFERING, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testAutomaticTradeRefund() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = DogecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = DogecoinACCTv3.buildTradeMessage(partner.getAddress(), dogecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + Block postDeploymentBlock = BlockUtils.mintBlock(repository); + int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); + + // Check refund + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in REFUNDED mode + CrossChainTradeData tradeData = DogecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.REFUNDED, tradeData.mode); + + // Test orphaning + BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance; + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretCorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = DogecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = DogecoinACCTv3.buildTradeMessage(partner.getAddress(), dogecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secret to AT, from correct account + messageData = DogecoinACCTv3.buildRedeemMessage(secretA, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in REDEEMED mode + CrossChainTradeData tradeData = DogecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.REDEEMED, tradeData.mode); + + // Check balances + long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee() + redeemAmount; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-redeem balance incorrect", expectedBalance, actualBalance); + + // Orphan redeem + BlockUtils.orphanLastBlock(repository); + + // Check balances + expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-orphan/pre-redeem balance incorrect", expectedBalance, actualBalance); + + // Check AT state + ATStateData postOrphanAtStateData = repository.getATRepository().getLatestATState(atAddress); + + assertTrue("AT states mismatch", Arrays.equals(preRedeemAtStateData.getStateData(), postOrphanAtStateData.getStateData())); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretIncorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = DogecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = DogecoinACCTv3.buildTradeMessage(partner.getAddress(), dogecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secret to AT, but from wrong account + messageData = DogecoinACCTv3.buildRedeemMessage(secretA, partner.getAddress()); + messageTransaction = sendMessage(repository, bystander, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + CrossChainTradeData tradeData = DogecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check balances + long expectedBalance = partnersInitialBalance; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Check eventual refund + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + } + } + + @SuppressWarnings("unused") + @Test + public void testIncorrectSecretCorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = DogecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = DogecoinACCTv3.buildTradeMessage(partner.getAddress(), dogecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send incorrect secret to AT, from correct account + byte[] wrongSecret = new byte[32]; + RANDOM.nextBytes(wrongSecret); + messageData = DogecoinACCTv3.buildRedeemMessage(wrongSecret, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + CrossChainTradeData tradeData = DogecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Check eventual refund + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretCorrectSenderInvalidMessageLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = DogecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = DogecoinACCTv3.buildTradeMessage(partner.getAddress(), dogecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secret to AT, from correct account, but missing receive address, hence incorrect length + messageData = Bytes.concat(secretA); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should be in TRADING mode + CrossChainTradeData tradeData = DogecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testDescribeDeployed() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + + List executableAts = repository.getATRepository().getAllExecutableATs(); + + for (ATData atData : executableAts) { + String atAddress = atData.getATAddress(); + byte[] codeBytes = atData.getCodeBytes(); + byte[] codeHash = Crypto.digest(codeBytes); + + System.out.println(String.format("%s: code length: %d byte%s, code hash: %s", + atAddress, + codeBytes.length, + (codeBytes.length != 1 ? "s": ""), + HashCode.fromBytes(codeHash))); + + // Not one of ours? + if (!Arrays.equals(codeHash, DogecoinACCTv3.CODE_BYTES_HASH)) + continue; + + describeAt(repository, atAddress); + } + } + } + + private int calcTestLockTimeA(long messageTimestamp) { + return (int) (messageTimestamp / 1000L + tradeTimeout * 60); + } + + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, String tradeAddress) throws DataException { + byte[] creationBytes = DogecoinACCTv3.buildQortalAT(tradeAddress, dogecoinPublicKeyHash, redeemAmount, dogecoinAmount, tradeTimeout); + + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "QORT-DOGE cross-chain trade"; + String description = String.format("Qortal-Dogecoin cross-chain trade"); + String atType = "ACCT"; + String tags = "QORT-DOGE ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, messageTransactionData, sender); + + return messageTransaction; + } + + private void checkTradeRefund(Repository repository, Account deployer, long deployersInitialBalance, long deployAtFee) throws DataException { + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + int refundTimeout = tradeTimeout / 2 + 1; // close enough + + // AT should automatically refund deployer after 'refundTimeout' blocks + for (int blockCount = 0; blockCount <= refundTimeout; ++blockCount) + BlockUtils.mintBlock(repository); + + // We don't bother to exactly calculate QORT spent running AT for several blocks, but we do know the expected range + long expectedMinimumBalance = deployersPostDeploymentBalance; + long expectedMaximumBalance = deployersInitialBalance - deployAtFee; + + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); + assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); + } + + private void describeAt(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = DogecoinACCTv3.getInstance().populateTradeData(repository, atData); + + Function epochMilliFormatter = (timestamp) -> LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneOffset.UTC).format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM)); + int currentBlockHeight = repository.getBlockRepository().getBlockchainHeight(); + + System.out.print(String.format("%s:\n" + + "\tmode: %s\n" + + "\tcreator: %s,\n" + + "\tcreation timestamp: %s,\n" + + "\tcurrent balance: %s QORT,\n" + + "\tis finished: %b,\n" + + "\tredeem payout: %s QORT,\n" + + "\texpected dogecoin: %s DOGE,\n" + + "\tcurrent block height: %d,\n", + tradeData.qortalAtAddress, + tradeData.mode, + tradeData.qortalCreator, + epochMilliFormatter.apply(tradeData.creationTimestamp), + Amounts.prettyAmount(tradeData.qortBalance), + atData.getIsFinished(), + Amounts.prettyAmount(tradeData.qortAmount), + Amounts.prettyAmount(tradeData.expectedForeignAmount), + currentBlockHeight)); + + if (tradeData.mode != AcctMode.OFFERING && tradeData.mode != AcctMode.CANCELLED) { + System.out.println(String.format("\trefund timeout: %d minutes,\n" + + "\trefund height: block %d,\n" + + "\tHASH160 of secret-A: %s,\n" + + "\tDogecoin P2SH-A nLockTime: %d (%s),\n" + + "\ttrade partner: %s\n" + + "\tpartner's receiving address: %s", + tradeData.refundTimeout, + tradeData.tradeRefundHeight, + HashCode.fromBytes(tradeData.hashOfSecretA).toString().substring(0, 40), + tradeData.lockTimeA, epochMilliFormatter.apply(tradeData.lockTimeA * 1000L), + tradeData.qortalPartnerAddress, + tradeData.qortalPartnerReceivingAddress)); + } + } + + private PrivateKeyAccount createTradeAccount(Repository repository) { + // We actually use a known test account with funds to avoid PoW compute + return Common.getTestAccount(repository, "alice"); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/DeployAT.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/DeployAT.java new file mode 100644 index 000000000..3a1f92089 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/litecoinv1/DeployAT.java @@ -0,0 +1,150 @@ +package org.qortal.test.crosschain.litecoinv1; + +import java.math.BigDecimal; + +import org.bitcoinj.core.ECKey; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.controller.Controller; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryFactory; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; +import org.qortal.test.crosschain.apps.Common; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Amounts; +import org.qortal.utils.Base58; + +import com.google.common.hash.HashCode; + +public class DeployAT { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: DeployAT ")); + System.err.println("A trading key-pair will be generated for you!"); + System.err.println(String.format("example: DeployAT " + + "7Eztjz2TsxwbrWUYEaSdLbASKQGTfK2rR7ViFc5gaiZw \\\n" + + "\t10 \\\n" + + "\t10.1 \\\n" + + "\t0.00864200 \\\n" + + "\t120")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 5) + usage(null); + + Common.init(); + + byte[] creatorPrivateKey = null; + long redeemAmount = 0; + long fundingAmount = 0; + long expectedLitecoin = 0; + int tradeTimeout = 0; + + int argIndex = 0; + try { + creatorPrivateKey = Base58.decode(args[argIndex++]); + if (creatorPrivateKey.length != 32) + usage("Refund private key must be 32 bytes"); + + redeemAmount = new BigDecimal(args[argIndex++]).setScale(8).unscaledValue().longValue(); + if (redeemAmount <= 0) + usage("QORT amount must be positive"); + + fundingAmount = new BigDecimal(args[argIndex++]).setScale(8).unscaledValue().longValue(); + if (fundingAmount <= redeemAmount) + usage("AT funding amount must be greater than QORT redeem amount"); + + expectedLitecoin = new BigDecimal(args[argIndex++]).setScale(8).unscaledValue().longValue(); + if (expectedLitecoin <= 0) + usage("Expected LTC amount must be positive"); + + tradeTimeout = Integer.parseInt(args[argIndex++]); + if (tradeTimeout < 60 || tradeTimeout > 50000) + usage("Trade timeout (minutes) must be between 60 and 50000"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + try { + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); + } catch (DataException e) { + System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); + System.exit(2); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount creatorAccount = new PrivateKeyAccount(repository, creatorPrivateKey); + System.out.println(String.format("Creator Qortal address: %s", creatorAccount.getAddress())); + System.out.println(String.format("QORT redeem amount: %s", Amounts.prettyAmount(redeemAmount))); + System.out.println(String.format("AT funding amount: %s", Amounts.prettyAmount(fundingAmount))); + + // Generate trading key-pair + byte[] tradePrivateKey = new ECKey().getPrivKeyBytes(); + PrivateKeyAccount tradeAccount = new PrivateKeyAccount(repository, tradePrivateKey); + byte[] litecoinPublicKeyHash = ECKey.fromPrivate(tradePrivateKey).getPubKeyHash(); + + System.out.println(String.format("Trade private key: %s", HashCode.fromBytes(tradePrivateKey))); + + // Deploy AT + byte[] creationBytes = LitecoinACCTv1.buildQortalAT(tradeAccount.getAddress(), litecoinPublicKeyHash, redeemAmount, expectedLitecoin, tradeTimeout); + System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); + + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = creatorAccount.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", creatorAccount.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "QORT-LTC cross-chain trade"; + String description = String.format("Qortal-Litecoin cross-chain trade"); + String atType = "ACCT"; + String tags = "QORT-LTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, creatorAccount.getPublicKey(), fee, null); + DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + deployAtTransaction.sign(creatorAccount); + + byte[] signedBytes = null; + try { + signedBytes = TransactionTransformer.toBytes(deployAtTransactionData); + } catch (TransformationException e) { + System.err.println(String.format("Unable to convert transaction to base58: %s", e.getMessage())); + System.exit(2); + } + + DeployAtTransaction.ensureATAddress(deployAtTransactionData); + String atAddress = deployAtTransactionData.getAtAddress(); + + System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); + + System.out.println(String.format("AT address: %s", atAddress)); + } catch (DataException e) { + System.err.println(String.format("Repository issue: %s", e.getMessage())); + System.exit(2); + } + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/LitecoinACCTv1Tests.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/LitecoinACCTv1Tests.java new file mode 100644 index 000000000..609ff5f31 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/litecoinv1/LitecoinACCTv1Tests.java @@ -0,0 +1,770 @@ +package org.qortal.test.crosschain.litecoinv1; + +import static org.junit.Assert.*; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.function.Function; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.Account; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.block.Block; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crosschain.AcctMode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.utils.Amounts; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; + +public class LitecoinACCTv1Tests extends Common { + + public static final byte[] secretA = "This string is exactly 32 bytes!".getBytes(); + public static final byte[] hashOfSecretA = Crypto.hash160(secretA); // daf59884b4d1aec8c1b17102530909ee43c0151a + public static final byte[] litecoinPublicKeyHash = HashCode.fromString("bb00bb11bb22bb33bb44bb55bb66bb77bb88bb99").asBytes(); + public static final int tradeTimeout = 20; // blocks + public static final long redeemAmount = 80_40200000L; + public static final long fundingAmount = 123_45600000L; + public static final long litecoinAmount = 864200L; // 0.00864200 LTC + + private static final Random RANDOM = new Random(); + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testCompile() { + PrivateKeyAccount tradeAccount = createTradeAccount(null); + + byte[] creationBytes = LitecoinACCTv1.buildQortalAT(tradeAccount.getAddress(), litecoinPublicKeyHash, redeemAmount, litecoinAmount, tradeTimeout); + assertNotNull(creationBytes); + + System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); + } + + @Test + public void testDeploy() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + + long expectedBalance = deployersInitialBalance - fundingAmount - deployAtTransaction.getTransactionData().getFee(); + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = fundingAmount; + actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); + + assertEquals("AT's post-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = partnersInitialBalance; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-deployment balance incorrect", expectedBalance, actualBalance); + + // Test orphaning + BlockUtils.orphanLastBlock(repository); + + expectedBalance = deployersInitialBalance; + actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = 0; + actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); + + assertEquals("AT's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = partnersInitialBalance; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testOfferCancel() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + // Send creator's address to AT, instead of typical partner's address + byte[] messageData = LitecoinACCTv1.getInstance().buildCancelMessage(deployer.getAddress()); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); + long messageFee = messageTransaction.getTransactionData().getFee(); + + // AT should process 'cancel' message in next block + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + + // Check balances + long expectedMinimumBalance = deployersPostDeploymentBalance; + long expectedMaximumBalance = deployersInitialBalance - deployAtFee - messageFee; + + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); + assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); + + // Test orphaning + BlockUtils.orphanLastBlock(repository); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance - messageFee; + actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testOfferCancelInvalidLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + // Instead of sending creator's address to AT, send too-short/invalid message + byte[] messageData = new byte[7]; + RANDOM.nextBytes(messageData); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); + long messageFee = messageTransaction.getTransactionData().getFee(); + + // AT should process 'cancel' message in next block + // As message is too short, it will be padded to 32bytes but cancel code doesn't care about message content, so should be ok + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testTradingInfoProcessing() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + Block postDeploymentBlock = BlockUtils.mintBlock(repository); + int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + describeAt(repository, atAddress); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + + // AT should be in TRADE mode + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check hashOfSecretA was extracted correctly + assertTrue(Arrays.equals(hashOfSecretA, tradeData.hashOfSecretA)); + + // Check trade partner Qortal address was extracted correctly + assertEquals(partner.getAddress(), tradeData.qortalPartnerAddress); + + // Check trade partner's Litecoin PKH was extracted correctly + assertTrue(Arrays.equals(litecoinPublicKeyHash, tradeData.partnerForeignPKH)); + + // Test orphaning + BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance; + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + // TEST SENDING TRADING INFO BUT NOT FROM AT CREATOR (SHOULD BE IGNORED) + @SuppressWarnings("unused") + @Test + public void testIncorrectTradeSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT BUT NOT FROM AT CREATOR + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, bystander, messageData, atAddress); + + BlockUtils.mintBlock(repository); + + long expectedBalance = partnersInitialBalance; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-initial-payout balance incorrect", expectedBalance, actualBalance); + + describeAt(repository, atAddress); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + + // AT should still be in OFFER mode + assertEquals(AcctMode.OFFERING, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testAutomaticTradeRefund() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + Block postDeploymentBlock = BlockUtils.mintBlock(repository); + int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); + + // Check refund + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in REFUNDED mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.REFUNDED, tradeData.mode); + + // Test orphaning + BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance; + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretCorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secret to AT, from correct account + messageData = LitecoinACCTv1.buildRedeemMessage(secretA, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in REDEEMED mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.REDEEMED, tradeData.mode); + + // Check balances + long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee() + redeemAmount; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-redeem balance incorrect", expectedBalance, actualBalance); + + // Orphan redeem + BlockUtils.orphanLastBlock(repository); + + // Check balances + expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-orphan/pre-redeem balance incorrect", expectedBalance, actualBalance); + + // Check AT state + ATStateData postOrphanAtStateData = repository.getATRepository().getLatestATState(atAddress); + + assertTrue("AT states mismatch", Arrays.equals(preRedeemAtStateData.getStateData(), postOrphanAtStateData.getStateData())); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretIncorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secret to AT, but from wrong account + messageData = LitecoinACCTv1.buildRedeemMessage(secretA, partner.getAddress()); + messageTransaction = sendMessage(repository, bystander, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check balances + long expectedBalance = partnersInitialBalance; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Check eventual refund + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + } + } + + @SuppressWarnings("unused") + @Test + public void testIncorrectSecretCorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send incorrect secret to AT, from correct account + byte[] wrongSecret = new byte[32]; + RANDOM.nextBytes(wrongSecret); + messageData = LitecoinACCTv1.buildRedeemMessage(wrongSecret, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Check eventual refund + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretCorrectSenderInvalidMessageLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secret to AT, from correct account, but missing receive address, hence incorrect length + messageData = Bytes.concat(secretA); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should be in TRADING mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testDescribeDeployed() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + + List executableAts = repository.getATRepository().getAllExecutableATs(); + + for (ATData atData : executableAts) { + String atAddress = atData.getATAddress(); + byte[] codeBytes = atData.getCodeBytes(); + byte[] codeHash = Crypto.digest(codeBytes); + + System.out.println(String.format("%s: code length: %d byte%s, code hash: %s", + atAddress, + codeBytes.length, + (codeBytes.length != 1 ? "s": ""), + HashCode.fromBytes(codeHash))); + + // Not one of ours? + if (!Arrays.equals(codeHash, LitecoinACCTv1.CODE_BYTES_HASH)) + continue; + + describeAt(repository, atAddress); + } + } + } + + private int calcTestLockTimeA(long messageTimestamp) { + return (int) (messageTimestamp / 1000L + tradeTimeout * 60); + } + + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, String tradeAddress) throws DataException { + byte[] creationBytes = LitecoinACCTv1.buildQortalAT(tradeAddress, litecoinPublicKeyHash, redeemAmount, litecoinAmount, tradeTimeout); + + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "QORT-LTC cross-chain trade"; + String description = String.format("Qortal-Litecoin cross-chain trade"); + String atType = "ACCT"; + String tags = "QORT-LTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, messageTransactionData, sender); + + return messageTransaction; + } + + private void checkTradeRefund(Repository repository, Account deployer, long deployersInitialBalance, long deployAtFee) throws DataException { + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + int refundTimeout = tradeTimeout / 2 + 1; // close enough + + // AT should automatically refund deployer after 'refundTimeout' blocks + for (int blockCount = 0; blockCount <= refundTimeout; ++blockCount) + BlockUtils.mintBlock(repository); + + // We don't bother to exactly calculate QORT spent running AT for several blocks, but we do know the expected range + long expectedMinimumBalance = deployersPostDeploymentBalance; + long expectedMaximumBalance = deployersInitialBalance - deployAtFee; + + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); + assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); + } + + private void describeAt(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + + Function epochMilliFormatter = (timestamp) -> LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneOffset.UTC).format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM)); + int currentBlockHeight = repository.getBlockRepository().getBlockchainHeight(); + + System.out.print(String.format("%s:\n" + + "\tmode: %s\n" + + "\tcreator: %s,\n" + + "\tcreation timestamp: %s,\n" + + "\tcurrent balance: %s QORT,\n" + + "\tis finished: %b,\n" + + "\tredeem payout: %s QORT,\n" + + "\texpected Litecoin: %s LTC,\n" + + "\tcurrent block height: %d,\n", + tradeData.qortalAtAddress, + tradeData.mode, + tradeData.qortalCreator, + epochMilliFormatter.apply(tradeData.creationTimestamp), + Amounts.prettyAmount(tradeData.qortBalance), + atData.getIsFinished(), + Amounts.prettyAmount(tradeData.qortAmount), + Amounts.prettyAmount(tradeData.expectedForeignAmount), + currentBlockHeight)); + + if (tradeData.mode != AcctMode.OFFERING && tradeData.mode != AcctMode.CANCELLED) { + System.out.println(String.format("\trefund timeout: %d minutes,\n" + + "\trefund height: block %d,\n" + + "\tHASH160 of secret-A: %s,\n" + + "\tLitecoin P2SH-A nLockTime: %d (%s),\n" + + "\ttrade partner: %s\n" + + "\tpartner's receiving address: %s", + tradeData.refundTimeout, + tradeData.tradeRefundHeight, + HashCode.fromBytes(tradeData.hashOfSecretA).toString().substring(0, 40), + tradeData.lockTimeA, epochMilliFormatter.apply(tradeData.lockTimeA * 1000L), + tradeData.qortalPartnerAddress, + tradeData.qortalPartnerReceivingAddress)); + } + } + + private PrivateKeyAccount createTradeAccount(Repository repository) { + // We actually use a known test account with funds to avoid PoW compute + return Common.getTestAccount(repository, "alice"); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/SendCancelMessage.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendCancelMessage.java new file mode 100644 index 000000000..2d04098cd --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendCancelMessage.java @@ -0,0 +1,90 @@ +package org.qortal.test.crosschain.litecoinv1; + +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.Controller; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crypto.Crypto; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryFactory; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; +import org.qortal.test.crosschain.apps.Common; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Base58; + +public class SendCancelMessage { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: SendCancelMessage ")); + System.err.println(String.format("example: SendCancelMessage " + + "7Eztjz2TsxwbrWUYEaSdLbASKQGTfK2rR7ViFc5gaiZw \\\n" + + "\tAaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 2) + usage(null); + + Common.init(); + + byte[] qortalPrivateKey = null; + String atAddress = null; + + int argIndex = 0; + try { + qortalPrivateKey = Base58.decode(args[argIndex++]); + if (qortalPrivateKey.length != 32) + usage("Refund private key must be 32 bytes"); + + atAddress = args[argIndex++]; + if (!Crypto.isValidAtAddress(atAddress)) + usage("Invalid AT address"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + try { + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); + } catch (DataException e) { + System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); + System.exit(2); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount qortalAccount = new PrivateKeyAccount(repository, qortalPrivateKey); + + String creatorQortalAddress = qortalAccount.getAddress(); + System.out.println(String.format("Qortal address: %s", creatorQortalAddress)); + + byte[] messageData = LitecoinACCTv1.getInstance().buildCancelMessage(creatorQortalAddress); + MessageTransaction messageTransaction = MessageTransaction.build(repository, qortalAccount, Group.NO_GROUP, atAddress, messageData, false, false); + + System.out.println("Computing nonce..."); + messageTransaction.computeNonce(); + messageTransaction.sign(qortalAccount); + + byte[] signedBytes = null; + try { + signedBytes = TransactionTransformer.toBytes(messageTransaction.getTransactionData()); + } catch (TransformationException e) { + System.err.println(String.format("Unable to convert transaction to bytes: %s", e.getMessage())); + System.exit(2); + } + + System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); + } catch (DataException e) { + System.err.println(String.format("Repository issue: %s", e.getMessage())); + System.exit(2); + } + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/SendRedeemMessage.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendRedeemMessage.java new file mode 100644 index 000000000..20386d2ae --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendRedeemMessage.java @@ -0,0 +1,101 @@ +package org.qortal.test.crosschain.litecoinv1; + +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.Controller; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crypto.Crypto; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryFactory; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; +import org.qortal.test.crosschain.apps.Common; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Base58; + +import com.google.common.hash.HashCode; + +public class SendRedeemMessage { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: SendRedeemMessage ")); + System.err.println(String.format("example: SendRedeemMessage " + + "dbfe739f5a3ecf7b0a22cea71f73d86ec71355b740e5972bcdf9e8bb4721ab9d \\\n" + + "\tAaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \\\n" + + "\t5468697320737472696e672069732065786163746c7920333220627974657321 \\\n" + + "\tQqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 4) + usage(null); + + Common.init(); + + byte[] tradePrivateKey = null; + String atAddress = null; + byte[] secret = null; + String receiveAddress = null; + + int argIndex = 0; + try { + tradePrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); + if (tradePrivateKey.length != 32) + usage("Refund private key must be 32 bytes"); + + atAddress = args[argIndex++]; + if (!Crypto.isValidAtAddress(atAddress)) + usage("Invalid AT address"); + + secret = HashCode.fromString(args[argIndex++]).asBytes(); + if (secret.length != 32) + usage("Secret must be 32 bytes"); + + receiveAddress = args[argIndex++]; + if (!Crypto.isValidAddress(receiveAddress)) + usage("Invalid Qortal receive address"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + try { + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); + } catch (DataException e) { + System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); + System.exit(2); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount tradeAccount = new PrivateKeyAccount(repository, tradePrivateKey); + + byte[] messageData = LitecoinACCTv1.buildRedeemMessage(secret, receiveAddress); + MessageTransaction messageTransaction = MessageTransaction.build(repository, tradeAccount, Group.NO_GROUP, atAddress, messageData, false, false); + + System.out.println("Computing nonce..."); + messageTransaction.computeNonce(); + messageTransaction.sign(tradeAccount); + + byte[] signedBytes = null; + try { + signedBytes = TransactionTransformer.toBytes(messageTransaction.getTransactionData()); + } catch (TransformationException e) { + System.err.println(String.format("Unable to convert transaction to bytes: %s", e.getMessage())); + System.exit(2); + } + + System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); + } catch (DataException e) { + System.err.println(String.format("Repository issue: %s", e.getMessage())); + System.exit(2); + } + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/SendTradeMessage.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendTradeMessage.java new file mode 100644 index 000000000..83e9a20e5 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendTradeMessage.java @@ -0,0 +1,118 @@ +package org.qortal.test.crosschain.litecoinv1; + +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.Controller; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crypto.Crypto; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryFactory; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; +import org.qortal.test.crosschain.apps.Common; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import com.google.common.hash.HashCode; + +public class SendTradeMessage { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: SendTradeMessage ")); + System.err.println(String.format("example: SendTradeMessage " + + "ed77aa2c62d785a9428725fc7f95b907be8a1cc43213239876a62cf70fdb6ecb \\\n" + + "\tAaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \\\n" + + "\tQqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq \\\n" + + "\tffffffffffffffffffffffffffffffffffffffff \\\n" + + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" + + "\t1600184800")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 6) + usage(null); + + Common.init(); + + byte[] tradePrivateKey = null; + String atAddress = null; + String partnerTradeAddress = null; + byte[] partnerTradePublicKeyHash = null; + byte[] hashOfSecret = null; + int lockTime = 0; + + int argIndex = 0; + try { + tradePrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); + if (tradePrivateKey.length != 32) + usage("Refund private key must be 32 bytes"); + + atAddress = args[argIndex++]; + if (!Crypto.isValidAtAddress(atAddress)) + usage("Invalid AT address"); + + partnerTradeAddress = args[argIndex++]; + if (!Crypto.isValidAddress(partnerTradeAddress)) + usage("Invalid partner trade Qortal address"); + + partnerTradePublicKeyHash = HashCode.fromString(args[argIndex++]).asBytes(); + if (partnerTradePublicKeyHash.length != 20) + usage("Partner trade PKH must be 20 bytes"); + + hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); + if (hashOfSecret.length != 20) + usage("HASH160 of secret must be 20 bytes"); + + lockTime = Integer.parseInt(args[argIndex++]); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + try { + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); + } catch (DataException e) { + System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); + System.exit(2); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount tradeAccount = new PrivateKeyAccount(repository, tradePrivateKey); + + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(NTP.getTime(), lockTime); + if (refundTimeout < 1) { + System.err.println("Refund timeout too small. Is locktime in the past?"); + System.exit(2); + } + + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partnerTradeAddress, partnerTradePublicKeyHash, hashOfSecret, lockTime, refundTimeout); + MessageTransaction messageTransaction = MessageTransaction.build(repository, tradeAccount, Group.NO_GROUP, atAddress, messageData, false, false); + + System.out.println("Computing nonce..."); + messageTransaction.computeNonce(); + messageTransaction.sign(tradeAccount); + + byte[] signedBytes = null; + try { + signedBytes = TransactionTransformer.toBytes(messageTransaction.getTransactionData()); + } catch (TransformationException e) { + System.err.println(String.format("Unable to convert transaction to bytes: %s", e.getMessage())); + System.exit(2); + } + + System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); + } catch (DataException e) { + System.err.println(String.format("Repository issue: %s", e.getMessage())); + System.exit(2); + } + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv3/LitecoinACCTv3Tests.java b/src/test/java/org/qortal/test/crosschain/litecoinv3/LitecoinACCTv3Tests.java new file mode 100644 index 000000000..009af5ea6 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/litecoinv3/LitecoinACCTv3Tests.java @@ -0,0 +1,769 @@ +package org.qortal.test.crosschain.litecoinv3; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.Account; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.block.Block; +import org.qortal.crosschain.AcctMode; +import org.qortal.crosschain.LitecoinACCTv3; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.utils.Amounts; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.function.Function; + +import static org.junit.Assert.*; + +public class LitecoinACCTv3Tests extends Common { + + public static final byte[] secretA = "This string is exactly 32 bytes!".getBytes(); + public static final byte[] hashOfSecretA = Crypto.hash160(secretA); // daf59884b4d1aec8c1b17102530909ee43c0151a + public static final byte[] litecoinPublicKeyHash = HashCode.fromString("bb00bb11bb22bb33bb44bb55bb66bb77bb88bb99").asBytes(); + public static final int tradeTimeout = 20; // blocks + public static final long redeemAmount = 80_40200000L; + public static final long fundingAmount = 123_45600000L; + public static final long litecoinAmount = 864200L; // 0.00864200 LTC + + private static final Random RANDOM = new Random(); + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testCompile() { + PrivateKeyAccount tradeAccount = createTradeAccount(null); + + byte[] creationBytes = LitecoinACCTv3.buildQortalAT(tradeAccount.getAddress(), litecoinPublicKeyHash, redeemAmount, litecoinAmount, tradeTimeout); + assertNotNull(creationBytes); + + System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); + } + + @Test + public void testDeploy() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + + long expectedBalance = deployersInitialBalance - fundingAmount - deployAtTransaction.getTransactionData().getFee(); + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = fundingAmount; + actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); + + assertEquals("AT's post-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = partnersInitialBalance; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-deployment balance incorrect", expectedBalance, actualBalance); + + // Test orphaning + BlockUtils.orphanLastBlock(repository); + + expectedBalance = deployersInitialBalance; + actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = 0; + actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); + + assertEquals("AT's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = partnersInitialBalance; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testOfferCancel() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + // Send creator's address to AT, instead of typical partner's address + byte[] messageData = LitecoinACCTv3.getInstance().buildCancelMessage(deployer.getAddress()); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); + long messageFee = messageTransaction.getTransactionData().getFee(); + + // AT should process 'cancel' message in next block + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + + // Check balances + long expectedMinimumBalance = deployersPostDeploymentBalance; + long expectedMaximumBalance = deployersInitialBalance - deployAtFee - messageFee; + + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); + assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); + + // Test orphaning + BlockUtils.orphanLastBlock(repository); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance - messageFee; + actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testOfferCancelInvalidLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + // Instead of sending creator's address to AT, send too-short/invalid message + byte[] messageData = new byte[7]; + RANDOM.nextBytes(messageData); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); + long messageFee = messageTransaction.getTransactionData().getFee(); + + // AT should process 'cancel' message in next block + // As message is too short, it will be padded to 32bytes but cancel code doesn't care about message content, so should be ok + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testTradingInfoProcessing() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv3.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + Block postDeploymentBlock = BlockUtils.mintBlock(repository); + int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + describeAt(repository, atAddress); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData); + + // AT should be in TRADE mode + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check hashOfSecretA was extracted correctly + assertTrue(Arrays.equals(hashOfSecretA, tradeData.hashOfSecretA)); + + // Check trade partner Qortal address was extracted correctly + assertEquals(partner.getAddress(), tradeData.qortalPartnerAddress); + + // Check trade partner's Litecoin PKH was extracted correctly + assertTrue(Arrays.equals(litecoinPublicKeyHash, tradeData.partnerForeignPKH)); + + // Test orphaning + BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance; + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + // TEST SENDING TRADING INFO BUT NOT FROM AT CREATOR (SHOULD BE IGNORED) + @SuppressWarnings("unused") + @Test + public void testIncorrectTradeSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT BUT NOT FROM AT CREATOR + byte[] messageData = LitecoinACCTv3.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, bystander, messageData, atAddress); + + BlockUtils.mintBlock(repository); + + long expectedBalance = partnersInitialBalance; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-initial-payout balance incorrect", expectedBalance, actualBalance); + + describeAt(repository, atAddress); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData); + + // AT should still be in OFFER mode + assertEquals(AcctMode.OFFERING, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testAutomaticTradeRefund() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv3.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + Block postDeploymentBlock = BlockUtils.mintBlock(repository); + int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); + + // Check refund + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in REFUNDED mode + CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.REFUNDED, tradeData.mode); + + // Test orphaning + BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance; + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretCorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv3.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secret to AT, from correct account + messageData = LitecoinACCTv3.buildRedeemMessage(secretA, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in REDEEMED mode + CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.REDEEMED, tradeData.mode); + + // Check balances + long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee() + redeemAmount; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-redeem balance incorrect", expectedBalance, actualBalance); + + // Orphan redeem + BlockUtils.orphanLastBlock(repository); + + // Check balances + expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-orphan/pre-redeem balance incorrect", expectedBalance, actualBalance); + + // Check AT state + ATStateData postOrphanAtStateData = repository.getATRepository().getLatestATState(atAddress); + + assertTrue("AT states mismatch", Arrays.equals(preRedeemAtStateData.getStateData(), postOrphanAtStateData.getStateData())); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretIncorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv3.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secret to AT, but from wrong account + messageData = LitecoinACCTv3.buildRedeemMessage(secretA, partner.getAddress()); + messageTransaction = sendMessage(repository, bystander, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check balances + long expectedBalance = partnersInitialBalance; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Check eventual refund + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + } + } + + @SuppressWarnings("unused") + @Test + public void testIncorrectSecretCorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv3.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send incorrect secret to AT, from correct account + byte[] wrongSecret = new byte[32]; + RANDOM.nextBytes(wrongSecret); + messageData = LitecoinACCTv3.buildRedeemMessage(wrongSecret, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Check eventual refund + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretCorrectSenderInvalidMessageLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv3.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv3.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secret to AT, from correct account, but missing receive address, hence incorrect length + messageData = Bytes.concat(secretA); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should be in TRADING mode + CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testDescribeDeployed() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + + List executableAts = repository.getATRepository().getAllExecutableATs(); + + for (ATData atData : executableAts) { + String atAddress = atData.getATAddress(); + byte[] codeBytes = atData.getCodeBytes(); + byte[] codeHash = Crypto.digest(codeBytes); + + System.out.println(String.format("%s: code length: %d byte%s, code hash: %s", + atAddress, + codeBytes.length, + (codeBytes.length != 1 ? "s": ""), + HashCode.fromBytes(codeHash))); + + // Not one of ours? + if (!Arrays.equals(codeHash, LitecoinACCTv3.CODE_BYTES_HASH)) + continue; + + describeAt(repository, atAddress); + } + } + } + + private int calcTestLockTimeA(long messageTimestamp) { + return (int) (messageTimestamp / 1000L + tradeTimeout * 60); + } + + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, String tradeAddress) throws DataException { + byte[] creationBytes = LitecoinACCTv3.buildQortalAT(tradeAddress, litecoinPublicKeyHash, redeemAmount, litecoinAmount, tradeTimeout); + + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "QORT-LTC cross-chain trade"; + String description = String.format("Qortal-Litecoin cross-chain trade"); + String atType = "ACCT"; + String tags = "QORT-LTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, messageTransactionData, sender); + + return messageTransaction; + } + + private void checkTradeRefund(Repository repository, Account deployer, long deployersInitialBalance, long deployAtFee) throws DataException { + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + int refundTimeout = tradeTimeout / 2 + 1; // close enough + + // AT should automatically refund deployer after 'refundTimeout' blocks + for (int blockCount = 0; blockCount <= refundTimeout; ++blockCount) + BlockUtils.mintBlock(repository); + + // We don't bother to exactly calculate QORT spent running AT for several blocks, but we do know the expected range + long expectedMinimumBalance = deployersPostDeploymentBalance; + long expectedMaximumBalance = deployersInitialBalance - deployAtFee; + + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); + assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); + } + + private void describeAt(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData); + + Function epochMilliFormatter = (timestamp) -> LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneOffset.UTC).format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM)); + int currentBlockHeight = repository.getBlockRepository().getBlockchainHeight(); + + System.out.print(String.format("%s:\n" + + "\tmode: %s\n" + + "\tcreator: %s,\n" + + "\tcreation timestamp: %s,\n" + + "\tcurrent balance: %s QORT,\n" + + "\tis finished: %b,\n" + + "\tredeem payout: %s QORT,\n" + + "\texpected Litecoin: %s LTC,\n" + + "\tcurrent block height: %d,\n", + tradeData.qortalAtAddress, + tradeData.mode, + tradeData.qortalCreator, + epochMilliFormatter.apply(tradeData.creationTimestamp), + Amounts.prettyAmount(tradeData.qortBalance), + atData.getIsFinished(), + Amounts.prettyAmount(tradeData.qortAmount), + Amounts.prettyAmount(tradeData.expectedForeignAmount), + currentBlockHeight)); + + if (tradeData.mode != AcctMode.OFFERING && tradeData.mode != AcctMode.CANCELLED) { + System.out.println(String.format("\trefund timeout: %d minutes,\n" + + "\trefund height: block %d,\n" + + "\tHASH160 of secret-A: %s,\n" + + "\tLitecoin P2SH-A nLockTime: %d (%s),\n" + + "\ttrade partner: %s\n" + + "\tpartner's receiving address: %s", + tradeData.refundTimeout, + tradeData.tradeRefundHeight, + HashCode.fromBytes(tradeData.hashOfSecretA).toString().substring(0, 40), + tradeData.lockTimeA, epochMilliFormatter.apply(tradeData.lockTimeA * 1000L), + tradeData.qortalPartnerAddress, + tradeData.qortalPartnerReceivingAddress)); + } + } + + private PrivateKeyAccount createTradeAccount(Repository repository) { + // We actually use a known test account with funds to avoid PoW compute + return Common.getTestAccount(repository, "alice"); + } + +} diff --git a/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java b/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java index d1062a790..88f63ddfa 100644 --- a/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java +++ b/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java @@ -7,7 +7,7 @@ import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; -import org.qortal.block.BlockMinter; +import org.qortal.controller.BlockMinter; import org.qortal.controller.Controller; import org.qortal.data.account.RewardShareData; import org.qortal.repository.DataException; diff --git a/src/test/java/org/qortal/test/minting/DisagreementTests.java b/src/test/java/org/qortal/test/minting/DisagreementTests.java index cd9724b8d..4dea75c73 100644 --- a/src/test/java/org/qortal/test/minting/DisagreementTests.java +++ b/src/test/java/org/qortal/test/minting/DisagreementTests.java @@ -10,7 +10,7 @@ import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; -import org.qortal.block.BlockMinter; +import org.qortal.controller.BlockMinter; import org.qortal.controller.Controller; import org.qortal.data.account.RewardShareData; import org.qortal.data.block.BlockData; diff --git a/src/test/java/org/qortal/test/minting/RewardShareTests.java b/src/test/java/org/qortal/test/minting/RewardShareTests.java index 25a4a38cc..cde3c2ff0 100644 --- a/src/test/java/org/qortal/test/minting/RewardShareTests.java +++ b/src/test/java/org/qortal/test/minting/RewardShareTests.java @@ -121,7 +121,7 @@ public void testNegativeInitialShareInvalid() throws DataException { Transaction transaction = Transaction.fromData(repository, transactionData); ValidationResult validationResult = transaction.isValidUnconfirmed(); - assertEquals("Initial 0% share should be invalid", ValidationResult.INVALID_REWARD_SHARE_PERCENT, validationResult); + assertNotSame("Creating reward-share with 'cancel' share-percent should be invalid", ValidationResult.OK, validationResult); } } diff --git a/src/test/java/org/qortal/test/minting/RewardTests.java b/src/test/java/org/qortal/test/minting/RewardTests.java index d023a1081..f7970ace7 100644 --- a/src/test/java/org/qortal/test/minting/RewardTests.java +++ b/src/test/java/org/qortal/test/minting/RewardTests.java @@ -7,15 +7,16 @@ import java.util.List; import java.util.Map; -import org.bitcoinj.core.Base58; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; import org.qortal.asset.Asset; import org.qortal.block.BlockChain; -import org.qortal.block.BlockMinter; import org.qortal.block.BlockChain.RewardByHeight; +import org.qortal.controller.BlockMinter; import org.qortal.data.account.AccountBalanceData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -25,9 +26,10 @@ import org.qortal.test.common.Common; import org.qortal.test.common.TestAccount; import org.qortal.utils.Amounts; +import org.qortal.utils.Base58; public class RewardTests extends Common { - + private static final Logger LOGGER = LogManager.getLogger(RewardTests.class); @Before public void beforeTest() throws DataException { Common.useDefaultSettings(); @@ -130,19 +132,19 @@ public void testLegacyQoraReward() throws DataException { /* * Example: - * + * * Block reward is 100 QORT, QORA-holders' share is 0.20 (20%) = 20 QORT - * + * * We hold 100 QORA * Someone else holds 28 QORA * Total QORA held: 128 QORA - * + * * Our portion of that is 100 QORA / 128 QORA * 20 QORT = 15.625 QORT - * + * * QORA holders earn at most 1 QORT per 250 QORA held. - * + * * So we can earn at most 100 QORA / 250 QORAperQORT = 0.4 QORT - * + * * Thus our block earning should be capped to 0.4 QORT. */ @@ -289,7 +291,7 @@ public void testNoFounderRewardScaling() throws DataException { * Dilbert is only account 'online'. * No founders online. * Some legacy QORA holders. - * + * * So Dilbert should receive 100% - legacy QORA holder's share. */ @@ -336,4 +338,462 @@ public void testLeftoverReward() throws DataException { } } -} \ No newline at end of file + /** Test rewards for level 1 and 2 accounts both pre and post the shareBinFix, including orphaning back through the feature trigger block */ + @Test + public void testLevel1And2Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + byte[] chloeRewardSharePrivateKey; + // Bob self-share NOT online + + // Chloe self share online + try { + chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + } catch (IllegalArgumentException ex) { + LOGGER.error("FAILED {}", ex.getLocalizedMessage(), ex); + throw ex; + } + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint a couple of blocks so that we are able to orphan them later + for (int i=0; i<2; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure that the levels are as we expect + assertEquals(1, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(2, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Ensure that only Alice is a founder + assertEquals(1, getFlags(repository, "alice")); + assertEquals(0, getFlags(repository, "bob")); + assertEquals(0, getFlags(repository, "chloe")); + assertEquals(0, getFlags(repository, "dilbert")); + + // Now that everyone is at level 1 or 2, we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are at the correct height and block reward value + assertEquals(6, (int) repository.getBlockRepository().getLastBlock().getHeight()); + assertEquals(10000000000L, blockReward); + + /* + * Alice, Chloe, and Dilbert are 'online'. Bob is offline. + * Chloe is level 1, Dilbert is level 2. + * One founder online (Alice, who is also level 1). + * No legacy QORA holders. + * + * Chloe and Dilbert should receive equal shares of the 5% block reward for Level 1 and 2 + * Alice should receive the remainder (95%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 1 and 2 to share the same reward (5%) + final int level1And2SharePercent = 5_00; // 5% + final long level1And2ShareAmount = (blockReward * level1And2SharePercent) / 100L / 100L; + final long expectedReward = level1And2ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderReward = blockReward - level1And2ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + assertEquals(500000000, level1And2ShareAmount); + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedReward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedReward); + + // Now orphan the latest block. This brings us to the threshold of the shareBinFix feature trigger. + BlockUtils.orphanBlocks(repository, 1); + assertEquals(5, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Ensure the latest post-fix block rewards have been subtracted and they have returned to their initial values + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance); + + // Orphan another block. This time, the block that was orphaned was prior to the shareBinFix feature trigger. + BlockUtils.orphanBlocks(repository, 1); + assertEquals(4, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Prior to the fix, the levels were incorrectly grouped + // Chloe should receive 100% of the level 1 reward, and Dilbert should receive 100% of the level 2+3 reward + final int level1SharePercent = 5_00; // 5% + final int level2And3SharePercent = 10_00; // 10% + final long level1ShareAmountBeforeFix = (blockReward * level1SharePercent) / 100L / 100L; + final long level2And3ShareAmountBeforeFix = (blockReward * level2And3SharePercent) / 100L / 100L; + final long expectedFounderRewardBeforeFix = blockReward - level1ShareAmountBeforeFix - level2And3ShareAmountBeforeFix; // Alice should receive the remainder + + // Validate the share amounts and balances + assertEquals(500000000, level1ShareAmountBeforeFix); + assertEquals(1000000000, level2And3ShareAmountBeforeFix); + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance-expectedFounderRewardBeforeFix); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance-level1ShareAmountBeforeFix); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance-level2And3ShareAmountBeforeFix); + + // Orphan the latest block one last time + BlockUtils.orphanBlocks(repository, 1); + assertEquals(3, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Validate balances + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance-(expectedFounderRewardBeforeFix*2)); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance-(level1ShareAmountBeforeFix*2)); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance-(level2And3ShareAmountBeforeFix*2)); + + } + } + + /** Test rewards for level 3 and 4 accounts */ + @Test + public void testLevel3And4Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share online + byte[] bobRewardSharePrivateKey = AccountUtils.rewardShare(repository, "bob", "bob", 0); + PrivateKeyAccount bobRewardShareAccount = new PrivateKeyAccount(repository, bobRewardSharePrivateKey); + mintingAndOnlineAccounts.add(bobRewardShareAccount); + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint enough blocks to bump testAccount levels to 3 and 4 + final int minterBlocksNeeded = cumulativeBlocksByLevel.get(4) - 20; // 20 blocks before level 4, so that the test accounts reach the correct levels + for (int bc = 0; bc < minterBlocksNeeded; ++bc) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure that the levels are as we expect + assertEquals(3, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(3, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(3, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(4, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Now that everyone is at level 3 or 4, we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are using the correct block reward value + assertEquals(100000000L, blockReward); + + /* + * Alice, Bob, Chloe, and Dilbert are 'online'. + * Bob and Chloe are level 3; Dilbert is level 4. + * One founder online (Alice, who is also level 3). + * No legacy QORA holders. + * + * Chloe, Bob and Dilbert should receive equal shares of the 10% block reward for level 3 and 4 + * Alice should receive the remainder (90%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 3 and 4 to share the same reward (10%) + final int level3And4SharePercent = 10_00; // 10% + final long level3And4ShareAmount = (blockReward * level3And4SharePercent) / 100L / 100L; + final long expectedReward = level3And4ShareAmount / 3; // The reward is split between Bob, Chloe, and Dilbert + final long expectedFounderReward = blockReward - level3And4ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance+expectedReward); + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedReward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedReward); + + } + } + + /** Test rewards for level 5 and 6 accounts */ + @Test + public void testLevel5And6Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share not initially online + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint enough blocks to bump testAccount levels to 5 and 6 + final int minterBlocksNeeded = cumulativeBlocksByLevel.get(6) - 20; // 20 blocks before level 6, so that the test accounts reach the correct levels + for (int bc = 0; bc < minterBlocksNeeded; ++bc) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Bob self-share now comes online + byte[] bobRewardSharePrivateKey = AccountUtils.rewardShare(repository, "bob", "bob", 0); + PrivateKeyAccount bobRewardShareAccount = new PrivateKeyAccount(repository, bobRewardSharePrivateKey); + mintingAndOnlineAccounts.add(bobRewardShareAccount); + + // Ensure that the levels are as we expect + assertEquals(5, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(5, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(6, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Now that everyone is at level 5 or 6 (except Bob who has only just started minting, so is at level 1), we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are using the correct block reward value + assertEquals(100000000L, blockReward); + + /* + * Alice, Bob, Chloe, and Dilbert are 'online'. + * Bob is level 1; Chloe is level 5; Dilbert is level 6. + * One founder online (Alice, who is also level 5). + * No legacy QORA holders. + * + * Chloe and Dilbert should receive equal shares of the 15% block reward for level 5 and 6 + * Bob should receive all of the level 1 and 2 reward (5%) + * Alice should receive the remainder (80%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 5 and 6 to share the same reward (15%) + final int level1And2SharePercent = 5_00; // 5% + final int level5And6SharePercent = 15_00; // 10% + final long level1And2ShareAmount = (blockReward * level1And2SharePercent) / 100L / 100L; + final long level5And6ShareAmount = (blockReward * level5And6SharePercent) / 100L / 100L; + final long expectedLevel1And2Reward = level1And2ShareAmount; // The reward is given entirely to Bob + final long expectedLevel5And6Reward = level5And6ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderReward = blockReward - level1And2ShareAmount - level5And6ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance+expectedLevel1And2Reward); + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedLevel5And6Reward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedLevel5And6Reward); + + } + } + + /** Test rewards for level 7 and 8 accounts */ + @Test + public void testLevel7And8Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share NOT online + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint enough blocks to bump testAccount levels to 7 and 8 + final int minterBlocksNeeded = cumulativeBlocksByLevel.get(8) - 20; // 20 blocks before level 8, so that the test accounts reach the correct levels + for (int bc = 0; bc < minterBlocksNeeded; ++bc) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure that the levels are as we expect + assertEquals(7, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(7, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(8, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Now that everyone is at level 7 or 8 (except Bob who has only just started minting, so is at level 1), we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are using the correct block reward value + assertEquals(100000000L, blockReward); + + /* + * Alice, Chloe, and Dilbert are 'online'. + * Chloe is level 7; Dilbert is level 8. + * One founder online (Alice, who is also level 7). + * No legacy QORA holders. + * + * Chloe and Dilbert should receive equal shares of the 20% block reward for level 7 and 8 + * Alice should receive the remainder (80%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 7 and 8 to share the same reward (20%) + final int level7And8SharePercent = 20_00; // 20% + final long level7And8ShareAmount = (blockReward * level7And8SharePercent) / 100L / 100L; + final long expectedLevel7And8Reward = level7And8ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderReward = blockReward - level7And8ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedLevel7And8Reward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedLevel7And8Reward); + + } + } + + /** Test rewards for level 9 and 10 accounts */ + @Test + public void testLevel9And10Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share not initially online + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint enough blocks to bump testAccount levels to 9 and 10 + final int minterBlocksNeeded = cumulativeBlocksByLevel.get(10) - 20; // 20 blocks before level 10, so that the test accounts reach the correct levels + for (int bc = 0; bc < minterBlocksNeeded; ++bc) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Bob self-share now comes online + byte[] bobRewardSharePrivateKey = AccountUtils.rewardShare(repository, "bob", "bob", 0); + PrivateKeyAccount bobRewardShareAccount = new PrivateKeyAccount(repository, bobRewardSharePrivateKey); + mintingAndOnlineAccounts.add(bobRewardShareAccount); + + // Ensure that the levels are as we expect + assertEquals(9, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(9, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(10, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Now that everyone is at level 7 or 8 (except Bob who has only just started minting, so is at level 1), we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are using the correct block reward value + assertEquals(100000000L, blockReward); + + /* + * Alice, Bob, Chloe, and Dilbert are 'online'. + * Bob is level 1; Chloe is level 9; Dilbert is level 10. + * One founder online (Alice, who is also level 9). + * No legacy QORA holders. + * + * Chloe and Dilbert should receive equal shares of the 25% block reward for level 9 and 10 + * Bob should receive all of the level 1 and 2 reward (5%) + * Alice should receive the remainder (70%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 9 and 10 to share the same reward (25%) + final int level1And2SharePercent = 5_00; // 5% + final int level9And10SharePercent = 25_00; // 25% + final long level1And2ShareAmount = (blockReward * level1And2SharePercent) / 100L / 100L; + final long level9And10ShareAmount = (blockReward * level9And10SharePercent) / 100L / 100L; + final long expectedLevel1And2Reward = level1And2ShareAmount; // The reward is given entirely to Bob + final long expectedLevel9And10Reward = level9And10ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderReward = blockReward - level1And2ShareAmount - level9And10ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance+expectedLevel1And2Reward); + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedLevel9And10Reward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedLevel9And10Reward); + + } + } + + + private int getFlags(Repository repository, String name) throws DataException { + TestAccount testAccount = Common.getTestAccount(repository, name); + return repository.getAccountRepository().getAccount(testAccount.getAddress()).getFlags(); + } + +} diff --git a/src/test/java/org/qortal/test/naming/BuySellTests.java b/src/test/java/org/qortal/test/naming/BuySellTests.java index f0320da53..cc014c251 100644 --- a/src/test/java/org/qortal/test/naming/BuySellTests.java +++ b/src/test/java/org/qortal/test/naming/BuySellTests.java @@ -20,7 +20,9 @@ import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; import org.qortal.utils.Amounts; +import org.qortal.utils.NTP; public class BuySellTests extends Common { @@ -62,6 +64,7 @@ public void afterTest() throws DataException { public void testRegisterName() throws DataException { // Register-name RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); String name = transactionData.getName(); diff --git a/src/test/java/org/qortal/test/naming/IntegrityTests.java b/src/test/java/org/qortal/test/naming/IntegrityTests.java new file mode 100644 index 000000000..c2232ec39 --- /dev/null +++ b/src/test/java/org/qortal/test/naming/IntegrityTests.java @@ -0,0 +1,451 @@ +package org.qortal.test.naming; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; +import org.qortal.data.transaction.*; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.utils.NTP; + +import java.util.List; + +import static org.junit.Assert.*; + +public class IntegrityTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testValidName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{\"age\":30}"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + + // Run the database integrity check for this name + NamesDatabaseIntegrityCheck integrityCheck = new NamesDatabaseIntegrityCheck(); + assertEquals(1, integrityCheck.rebuildName(name, repository)); + + // Ensure the name still exists and the data is still correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + } + } + + @Test + public void testBlankReducedName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "\uD83E\uDD73"; // Translates to a reducedName of "" + String data = "\uD83E\uDD73"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + + // Ensure the reducedName is blank + assertEquals("", repository.getNameRepository().fromName(name).getReducedName()); + + // Run the database integrity check for this name + NamesDatabaseIntegrityCheck integrityCheck = new NamesDatabaseIntegrityCheck(); + assertEquals(1, integrityCheck.rebuildName(name, repository)); + + // Ensure the name still exists and the data is still correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + assertEquals("", repository.getNameRepository().fromName(name).getReducedName()); + } + } + + @Test + public void testUpdateWithBlankNewName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name to Alice + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "initial_name"; + String data = "initial_data"; + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Update the name, but keep the new name blank + String newName = ""; + String newData = "updated_data"; + UpdateNameTransactionData updateTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), name, newName, newData); + TransactionUtils.signAndMint(repository, updateTransactionData, alice); + + // Ensure the original name exists and the data is correct + assertEquals(name, repository.getNameRepository().fromName(name).getName()); + assertEquals(newData, repository.getNameRepository().fromName(name).getData()); + + // Run the database integrity check for this name + NamesDatabaseIntegrityCheck integrityCheck = new NamesDatabaseIntegrityCheck(); + assertEquals(2, integrityCheck.rebuildName(name, repository)); + + // Ensure the name still exists and the data is still correct + assertEquals(name, repository.getNameRepository().fromName(name).getName()); + assertEquals(newData, repository.getNameRepository().fromName(name).getData()); + } + } + + @Test + public void testUpdateWithBlankNewNameAndBlankEmojiName() throws DataException { + // Attempt to simulate a real world problem where an emoji with blank reducedName + // confused the integrity check by associating it with previous UPDATE_NAME transactions + // due to them also having a blank "newReducedName" + + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name to Alice + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "initial_name"; + String data = "initial_data"; + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Update the name, but keep the new name blank + String newName = ""; + String newData = "updated_data"; + UpdateNameTransactionData updateTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), name, newName, newData); + TransactionUtils.signAndMint(repository, updateTransactionData, alice); + + // Register emoji name + String emojiName = "\uD83E\uDD73"; // Translates to a reducedName of "" + + // Ensure that the initial_name isn't associated with the emoji name + NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck(); + List transactions = namesDatabaseIntegrityCheck.fetchAllTransactionsInvolvingName(emojiName, repository); + assertEquals(0, transactions.size()); + } + } + + @Test + public void testMissingName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{\"age\":30}"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + + // Now delete the name, to simulate a database inconsistency + repository.getNameRepository().delete(name); + + // Ensure the name doesn't exist + assertNull(repository.getNameRepository().fromName(name)); + + // Run the database integrity check for this name and check that a row was modified + NamesDatabaseIntegrityCheck integrityCheck = new NamesDatabaseIntegrityCheck(); + assertEquals(1, integrityCheck.rebuildName(name, repository)); + + // Ensure the name exists again and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + } + } + + @Test + public void testMissingNameAfterUpdate() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{\"age\":30}"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + + // Update the name + String newData = "{\"age\":31}"; + UpdateNameTransactionData updateTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), name, name, newData); + TransactionUtils.signAndMint(repository, updateTransactionData, alice); + + // Ensure the name still exists and the data has been updated + assertEquals(newData, repository.getNameRepository().fromName(name).getData()); + + // Now delete the name, to simulate a database inconsistency + repository.getNameRepository().delete(name); + + // Ensure the name doesn't exist + assertNull(repository.getNameRepository().fromName(name)); + + // Run the database integrity check for this name + // We expect 2 modifications to be made - the original register name followed by the update + NamesDatabaseIntegrityCheck integrityCheck = new NamesDatabaseIntegrityCheck(); + assertEquals(2, integrityCheck.rebuildName(name, repository)); + + // Ensure the name exists and the data is correct + assertEquals(newData, repository.getNameRepository().fromName(name).getData()); + } + } + + @Test + public void testMissingNameAfterRename() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{\"age\":30}"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + + // Rename the name + String newName = "new-name"; + String newData = "{\"age\":31}"; + UpdateNameTransactionData updateTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), name, newName, newData); + TransactionUtils.signAndMint(repository, updateTransactionData, alice); + + // Ensure the new name exists and the data has been updated + assertEquals(newData, repository.getNameRepository().fromName(newName).getData()); + + // Ensure the old name doesn't exist + assertNull(repository.getNameRepository().fromName(name)); + + // Now delete the new name, to simulate a database inconsistency + repository.getNameRepository().delete(newName); + + // Ensure the new name doesn't exist + assertNull(repository.getNameRepository().fromName(newName)); + + // Attempt to register the new name + transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), newName, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + Transaction transaction = Transaction.fromData(repository, transactionData); + transaction.sign(alice); + + // Transaction should be invalid, because the database inconsistency was fixed by RegisterNameTransaction.preProcess() + Transaction.ValidationResult result = transaction.importAsUnconfirmed(); + assertTrue("Transaction should be invalid", Transaction.ValidationResult.OK != result); + assertTrue("Name should already be registered", Transaction.ValidationResult.NAME_ALREADY_REGISTERED == result); + } + } + + @Test + public void testRegisterMissingName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{\"age\":30}"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + + // Now delete the name, to simulate a database inconsistency + repository.getNameRepository().delete(name); + + // Ensure the name doesn't exist + assertNull(repository.getNameRepository().fromName(name)); + + // Attempt to register the name again + String duplicateName = "TEST-nÁme"; + transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), duplicateName, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + Transaction transaction = Transaction.fromData(repository, transactionData); + transaction.sign(alice); + + // Transaction should be invalid, because the database inconsistency was fixed by RegisterNameTransaction.preProcess() + Transaction.ValidationResult result = transaction.importAsUnconfirmed(); + assertTrue("Transaction should be invalid", Transaction.ValidationResult.OK != result); + assertTrue("Name should already be registered", Transaction.ValidationResult.NAME_ALREADY_REGISTERED == result); + } + } + + @Test + public void testUpdateMissingName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String initialName = "test-name"; + String data = "{\"age\":30}"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(initialName).getData()); + + // Now delete the name, to simulate a database inconsistency + repository.getNameRepository().delete(initialName); + + // Ensure the name doesn't exist + assertNull(repository.getNameRepository().fromName(initialName)); + + // Attempt to update the name + String newName = "new-name"; + String newData = ""; + TransactionData updateTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), initialName, newName, newData); + Transaction transaction = Transaction.fromData(repository, updateTransactionData); + transaction.sign(alice); + + // Transaction should be valid, because the database inconsistency was fixed by UpdateNameTransaction.preProcess() + Transaction.ValidationResult result = transaction.importAsUnconfirmed(); + assertTrue("Transaction should be valid", Transaction.ValidationResult.OK == result); + } + } + + @Test + public void testUpdateToMissingName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String initialName = "test-name"; + String data = "{\"age\":30}"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(initialName).getData()); + + // Register the second name that we will ultimately try and rename the first name to + String secondName = "new-missing-name"; + String secondNameData = "{\"data2\":true}"; + transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), secondName, secondNameData); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the second name exists and the data is correct + assertEquals(secondNameData, repository.getNameRepository().fromName(secondName).getData()); + + // Now delete the second name, to simulate a database inconsistency + repository.getNameRepository().delete(secondName); + + // Ensure the second name doesn't exist + assertNull(repository.getNameRepository().fromName(secondName)); + + // Attempt to rename the first name to the second name + TransactionData updateTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), initialName, secondName, secondNameData); + Transaction transaction = Transaction.fromData(repository, updateTransactionData); + transaction.sign(alice); + + // Transaction should be invalid, because the database inconsistency was fixed by UpdateNameTransaction.preProcess() + // Therefore the name that we are trying to rename TO already exists + Transaction.ValidationResult result = transaction.importAsUnconfirmed(); + assertTrue("Transaction should be invalid", Transaction.ValidationResult.OK != result); + assertTrue("Destination name should already exist", Transaction.ValidationResult.NAME_ALREADY_REGISTERED == result); + } + } + + @Test + public void testSellMissingName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{\"age\":30}"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + + // Now delete the name, to simulate a database inconsistency + repository.getNameRepository().delete(name); + + // Ensure the name doesn't exist + assertNull(repository.getNameRepository().fromName(name)); + + // Attempt to sell the name + TransactionData sellTransactionData = new SellNameTransactionData(TestTransaction.generateBase(alice), name, 123456); + Transaction transaction = Transaction.fromData(repository, sellTransactionData); + transaction.sign(alice); + + // Transaction should be valid, because the database inconsistency was fixed by SellNameTransaction.preProcess() + Transaction.ValidationResult result = transaction.importAsUnconfirmed(); + assertTrue("Transaction should be valid", Transaction.ValidationResult.OK == result); + } + } + + @Test + public void testBuyMissingName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{\"age\":30}"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + + // Now delete the name, to simulate a database inconsistency + repository.getNameRepository().delete(name); + + // Ensure the name doesn't exist + assertNull(repository.getNameRepository().fromName(name)); + + // Attempt to sell the name + long amount = 123456; + TransactionData sellTransactionData = new SellNameTransactionData(TestTransaction.generateBase(alice), name, amount); + TransactionUtils.signAndMint(repository, sellTransactionData, alice); + + // Ensure the name now exists + assertNotNull(repository.getNameRepository().fromName(name)); + + // Now delete the name again, to simulate another database inconsistency + repository.getNameRepository().delete(name); + + // Ensure the name doesn't exist + assertNull(repository.getNameRepository().fromName(name)); + + // Bob now attempts to buy the name + String seller = alice.getAddress(); + PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); + TransactionData buyTransactionData = new BuyNameTransactionData(TestTransaction.generateBase(bob), name, amount, seller); + Transaction transaction = Transaction.fromData(repository, buyTransactionData); + transaction.sign(bob); + + // Transaction should be valid, because the database inconsistency was fixed by SellNameTransaction.preProcess() + Transaction.ValidationResult result = transaction.importAsUnconfirmed(); + assertTrue("Transaction should be valid", Transaction.ValidationResult.OK == result); + } + } + +} diff --git a/src/test/java/org/qortal/test/naming/MiscTests.java b/src/test/java/org/qortal/test/naming/MiscTests.java index c46cbfabb..bdf3df9fc 100644 --- a/src/test/java/org/qortal/test/naming/MiscTests.java +++ b/src/test/java/org/qortal/test/naming/MiscTests.java @@ -3,21 +3,26 @@ import static org.junit.Assert.*; import java.util.List; +import java.util.Optional; +import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; -import org.qortal.data.transaction.RegisterNameTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.data.transaction.UpdateNameTransactionData; +import org.qortal.block.BlockChain; +import org.qortal.controller.BlockMinter; +import org.qortal.data.transaction.*; +import org.qortal.naming.Name; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; -import org.qortal.test.common.Common; -import org.qortal.test.common.TransactionUtils; +import org.qortal.settings.Settings; +import org.qortal.test.common.*; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.utils.NTP; public class MiscTests extends Common { @@ -32,9 +37,10 @@ public void testGetRecentNames() throws DataException { // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String name = "initial-name"; - String data = "initial-data"; + String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); List recentNames = repository.getNameRepository().getRecentNames(0L); @@ -51,14 +57,42 @@ public void testDuplicateRegisterName() throws DataException { // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String name = "test-name"; - String data = "{}"; + String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // duplicate String duplicateName = "TEST-nÁme"; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), duplicateName, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + Transaction transaction = Transaction.fromData(repository, transactionData); + transaction.sign(alice); + + ValidationResult result = transaction.importAsUnconfirmed(); + assertTrue("Transaction should be invalid", ValidationResult.OK != result); + } + } + + // test trying to register same name twice (with different creator) + @Test + public void testDuplicateRegisterNameWithDifferentCreator() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{}"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // duplicate (this time registered by Bob) + PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); + String duplicateName = "TEST-nÁme"; + transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), duplicateName, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); @@ -74,15 +108,17 @@ public void testUpdateToExistingName() throws DataException { // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String name = "test-name"; - String data = "{}"; + String data = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Register another name that we will later attempt to rename to first name (above) String otherName = "new-name"; String otherData = ""; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), otherName, otherData); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // we shouldn't be able to update name to existing name @@ -103,9 +139,10 @@ public void testRegisterAddressAsName() throws DataException { // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String name = alice.getAddress(); - String data = "{}"; + String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); @@ -121,9 +158,10 @@ public void testUpdateToAddressAsName() throws DataException { // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String name = "test-name"; - String data = "{}"; + String data = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // we shouldn't be able to update name to an address @@ -138,4 +176,201 @@ public void testUpdateToAddressAsName() throws DataException { } } + // test registering and then orphaning + @Test + public void testRegisterNameAndOrphan() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{\"age\":30}"; + + // Ensure the name doesn't exist + assertNull(repository.getNameRepository().fromName(name)); + + // Register the name + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + + // Orphan the latest block + BlockUtils.orphanBlocks(repository, 1); + + // Ensure the name doesn't exist once again + assertNull(repository.getNameRepository().fromName(name)); + } + } + + @Test + public void testOrphanAndReregisterName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{\"age\":30}"; + + // Ensure the name doesn't exist + assertNull(repository.getNameRepository().fromName(name)); + + // Register the name + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + + // Orphan the latest block + BlockUtils.orphanBlocks(repository, 1); + + // Ensure the name doesn't exist once again + assertNull(repository.getNameRepository().fromName(name)); + + // Now check there is an unconfirmed transaction + assertEquals(1, repository.getTransactionRepository().getUnconfirmedTransactions().size()); + + // Re-mint the block, including the original transaction + BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share")); + + // There should no longer be an unconfirmed transaction + assertEquals(0, repository.getTransactionRepository().getUnconfirmedTransactions().size()); + + // Orphan the latest block + BlockUtils.orphanBlocks(repository, 1); + + // There should now be an unconfirmed transaction again + assertEquals(1, repository.getTransactionRepository().getUnconfirmedTransactions().size()); + + // Re-mint the block, including the original transaction + BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share")); + + // Ensure there are no unconfirmed transactions + assertEquals(0, repository.getTransactionRepository().getUnconfirmedTransactions().size()); + } + } + + // test registering and then orphaning multiple times, with a different versions of the transaction each time + // we can sometimes end up with more than one version of a transaction, if it is signed and submitted twice + @Test + public void testMultipleRegisterNameAndOrphan() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{\"age\":30}"; + + for (int i = 1; i <= 10; i++) { + + // Ensure the name doesn't exist + assertNull(repository.getNameRepository().fromName(name)); + + // Register the name + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + + // The number of unconfirmed transactions should equal the number of cycles minus 1 (because one is in a block) + // If more than one made it into a block, this test would fail + assertEquals(i-1, repository.getTransactionRepository().getUnconfirmedTransactions().size()); + + // Orphan the latest block + BlockUtils.orphanBlocks(repository, 1); + + // The number of unconfirmed transactions should equal the number of cycles + assertEquals(i, repository.getTransactionRepository().getUnconfirmedTransactions().size()); + + // Ensure the name doesn't exist once again + assertNull(repository.getNameRepository().fromName(name)); + } + } + } + + @Test + public void testSaveName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + for (int i=0; i<10; i++) { + + String name = "test-name"; + String data = "{\"age\":30}"; + + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + + // Ensure the name doesn't exist + assertNull(repository.getNameRepository().fromName(name)); + + // Register the name + Name nameObj = new Name(repository, transactionData); + nameObj.register(); + + // Ensure the name now exists + assertNotNull(repository.getNameRepository().fromName(name)); + + // Unregister the name + nameObj.unregister(); + + // Ensure the name doesn't exist again + assertNull(repository.getNameRepository().fromName(name)); + + } + } + } + + // test name registration fee increase + @Test + public void testRegisterNameFeeIncrease() throws DataException, IllegalAccessException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Set nameRegistrationUnitFeeTimestamp to a time far in the future + long futureTimestamp = 9999999999999L; // 20 Nov 2286 + FieldUtils.writeField(BlockChain.getInstance(), "nameRegistrationUnitFeeTimestamp", futureTimestamp, true); + assertEquals(futureTimestamp, BlockChain.getInstance().getNameRegistrationUnitFeeTimestamp()); + + // Validate unit fees pre and post timestamp + assertEquals(10000000, BlockChain.getInstance().getUnitFee()); // 0.1 QORT + assertEquals(500000000, BlockChain.getInstance().getNameRegistrationUnitFee()); // 5 QORT + + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{\"age\":30}"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + assertEquals(10000000L, transactionData.getFee().longValue()); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Set nameRegistrationUnitFeeTimestamp to a time in the past + Long now = NTP.getTime(); + FieldUtils.writeField(BlockChain.getInstance(), "nameRegistrationUnitFeeTimestamp", now - 1000L, true); + assertEquals(now - 1000L, BlockChain.getInstance().getNameRegistrationUnitFeeTimestamp()); + + // Register a different name + // First try with the default unit fee + String name2 = "test-name-2"; + transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, data); + assertEquals(10000000L, transactionData.getFee().longValue()); + Transaction transaction = Transaction.fromData(repository, transactionData); + transaction.sign(alice); + ValidationResult result = transaction.importAsUnconfirmed(); + assertTrue("Transaction should be invalid", ValidationResult.INSUFFICIENT_FEE == result); + + // Now try using correct fee (this is specified by the UI, via the /transaction/unitfee API endpoint) + transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; + assertEquals(500000000L, transactionData.getFee().longValue()); + transaction = Transaction.fromData(repository, transactionData); + transaction.sign(alice); + result = transaction.importAsUnconfirmed(); + assertTrue("Transaction should be valid", ValidationResult.OK == result); + } + } + } diff --git a/src/test/java/org/qortal/test/naming/UpdateTests.java b/src/test/java/org/qortal/test/naming/UpdateTests.java index ffbf7177f..d591b3f32 100644 --- a/src/test/java/org/qortal/test/naming/UpdateTests.java +++ b/src/test/java/org/qortal/test/naming/UpdateTests.java @@ -5,6 +5,7 @@ import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.data.naming.NameData; import org.qortal.data.transaction.RegisterNameTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.UpdateNameTransactionData; @@ -15,6 +16,8 @@ import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.utils.NTP; public class UpdateTests extends Common { @@ -29,12 +32,22 @@ public void testUpdateName() throws DataException { // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String initialName = "initial-name"; - String initialData = "initial-data"; + String initialReducedName = "initia1-name"; + String initialData = "{\"age\":30}"; TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(initialTransactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, initialTransactionData, alice); + // Check name, reduced name, and data exist + assertTrue(repository.getNameRepository().nameExists(initialName)); + NameData nameData = repository.getNameRepository().fromName(initialName); + assertEquals("initia1-name", nameData.getReducedName()); + assertEquals(initialData, nameData.getData()); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); + String newName = "new-name"; + String newReducedName = "new-name"; String newData = ""; TransactionData updateTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), initialName, newName, newData); TransactionUtils.signAndMint(repository, updateTransactionData, alice); @@ -42,20 +55,37 @@ public void testUpdateName() throws DataException { // Check old name no longer exists assertFalse(repository.getNameRepository().nameExists(initialName)); + // Check reduced name no longer exists + assertNull(repository.getNameRepository().fromReducedName(initialReducedName)); + // Check new name exists assertTrue(repository.getNameRepository().nameExists(newName)); + // Check reduced name and data are correct for new name + NameData newNameData = repository.getNameRepository().fromName(newReducedName); + assertEquals(newReducedName, newNameData.getReducedName()); + // Data should remain the same because it was empty in the UpdateNameTransactionData + assertEquals(initialData, newNameData.getData()); + // Check updated timestamp is correct assertEquals((Long) updateTransactionData.getTimestamp(), repository.getNameRepository().fromName(newName).getUpdated()); // orphan and recheck BlockUtils.orphanLastBlock(repository); - // Check new name no longer exists + // Check new name and reduced name no longer exist assertFalse(repository.getNameRepository().nameExists(newName)); + assertNull(repository.getNameRepository().fromReducedName(newReducedName)); - // Check old name exists again + // Check old name and reduced name exist again assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); + + // Check data and reduced name are still present for this name + assertTrue(repository.getNameRepository().nameExists(initialName)); + nameData = repository.getNameRepository().fromName(initialName); + assertEquals(initialReducedName, nameData.getReducedName()); + assertEquals(initialData, nameData.getData()); // Check updated timestamp is empty assertNull(repository.getNameRepository().fromName(initialName).getUpdated()); @@ -68,11 +98,18 @@ public void testUpdateNameSameOwner() throws DataException { // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String initialName = "initial-name"; - String initialData = "initial-data"; + String initialData = "{\"age\":30}"; + + String constantReducedName = "initia1-name"; TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(initialTransactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, initialTransactionData, alice); + // Check initial name exists + assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(constantReducedName)); + String newName = "Initial-Name"; String newData = ""; TransactionData updateTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), initialName, newName, newData); @@ -83,6 +120,7 @@ public void testUpdateNameSameOwner() throws DataException { // Check new name exists assertTrue(repository.getNameRepository().nameExists(newName)); + assertNotNull(repository.getNameRepository().fromReducedName(constantReducedName)); // Check updated timestamp is correct assertEquals((Long) updateTransactionData.getTimestamp(), repository.getNameRepository().fromName(newName).getUpdated()); @@ -95,6 +133,7 @@ public void testUpdateNameSameOwner() throws DataException { // Check old name exists again assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(constantReducedName)); // Check updated timestamp is empty assertNull(repository.getNameRepository().fromName(initialName).getUpdated()); @@ -108,32 +147,44 @@ public void testDoubleUpdateName() throws DataException { // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String initialName = "initial-name"; - String initialData = "initial-data"; + String initialReducedName = "initia1-name"; + String initialData = "{\"age\":30}"; TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(initialTransactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, initialTransactionData, alice); + // Check initial name exists + assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); + String middleName = "middle-name"; + String middleReducedName = "midd1e-name"; String middleData = ""; TransactionData middleTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), initialName, middleName, middleData); TransactionUtils.signAndMint(repository, middleTransactionData, alice); // Check old name no longer exists assertFalse(repository.getNameRepository().nameExists(initialName)); + assertNull(repository.getNameRepository().fromReducedName(initialReducedName)); // Check new name exists assertTrue(repository.getNameRepository().nameExists(middleName)); + assertNotNull(repository.getNameRepository().fromReducedName(middleReducedName)); String newestName = "newest-name"; + String newestReducedName = "newest-name"; String newestData = "newest-data"; TransactionData newestTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), middleName, newestName, newestData); TransactionUtils.signAndMint(repository, newestTransactionData, alice); // Check previous name no longer exists assertFalse(repository.getNameRepository().nameExists(middleName)); + assertNull(repository.getNameRepository().fromReducedName(middleReducedName)); // Check newest name exists assertTrue(repository.getNameRepository().nameExists(newestName)); + assertNotNull(repository.getNameRepository().fromReducedName(newestReducedName)); // Check updated timestamp is correct assertEquals((Long) newestTransactionData.getTimestamp(), repository.getNameRepository().fromName(newestName).getUpdated()); @@ -143,9 +194,11 @@ public void testDoubleUpdateName() throws DataException { // Check newest name no longer exists assertFalse(repository.getNameRepository().nameExists(newestName)); + assertNull(repository.getNameRepository().fromReducedName(newestReducedName)); // Check previous name exists again assertTrue(repository.getNameRepository().nameExists(middleName)); + assertNotNull(repository.getNameRepository().fromReducedName(middleReducedName)); // Check updated timestamp is correct assertEquals((Long) middleTransactionData.getTimestamp(), repository.getNameRepository().fromName(middleName).getUpdated()); @@ -155,9 +208,11 @@ public void testDoubleUpdateName() throws DataException { // Check new name no longer exists assertFalse(repository.getNameRepository().nameExists(middleName)); + assertNull(repository.getNameRepository().fromReducedName(middleReducedName)); // Check original name exists again assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); // Check updated timestamp is empty assertNull(repository.getNameRepository().fromName(initialName).getUpdated()); @@ -171,11 +226,17 @@ public void testIntermediateUpdateName() throws DataException { // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String initialName = "initial-name"; - String initialData = "initial-data"; + String initialReducedName = "initia1-name"; + String initialData = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); + // Check initial name exists + assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); + // Don't update name, but update data. // This tests whether reverting a future update/sale can find the correct previous name String middleName = ""; @@ -185,29 +246,35 @@ public void testIntermediateUpdateName() throws DataException { // Check old name still exists assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); String newestName = "newest-name"; + String newestReducedName = "newest-name"; String newestData = "newest-data"; transactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), initialName, newestName, newestData); TransactionUtils.signAndMint(repository, transactionData, alice); // Check previous name no longer exists assertFalse(repository.getNameRepository().nameExists(initialName)); + assertNull(repository.getNameRepository().fromReducedName(initialReducedName)); // Check newest name exists assertTrue(repository.getNameRepository().nameExists(newestName)); + assertNotNull(repository.getNameRepository().fromReducedName(newestReducedName)); // orphan and recheck BlockUtils.orphanLastBlock(repository); // Check original name exists again assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); // orphan and recheck BlockUtils.orphanLastBlock(repository); // Check original name still exists assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); } } @@ -217,11 +284,17 @@ public void testUpdateData() throws DataException { // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String initialName = "initial-name"; - String initialData = "initial-data"; + String initialReducedName = "initia1-name"; + String initialData = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); + // Check initial name exists + assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); + String newName = ""; String newData = "new-data"; transactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), initialName, newName, newData); @@ -229,6 +302,7 @@ public void testUpdateData() throws DataException { // Check name still exists assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); // Check data is correct assertEquals(newData, repository.getNameRepository().fromName(initialName).getData()); @@ -238,6 +312,7 @@ public void testUpdateData() throws DataException { // Check name still exists assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); // Check old data restored assertEquals(initialData, repository.getNameRepository().fromName(initialName).getData()); @@ -251,13 +326,20 @@ public void testDoubleUpdateData() throws DataException { // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String initialName = "initial-name"; - String initialData = "initial-data"; + String initialReducedName = "initia1-name"; + String initialData = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); + // Check initial name exists + assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); + // Update data String middleName = "middle-name"; + String middleReducedName = "midd1e-name"; String middleData = "middle-data"; transactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), initialName, middleName, middleData); TransactionUtils.signAndMint(repository, transactionData, alice); @@ -266,6 +348,7 @@ public void testDoubleUpdateData() throws DataException { assertEquals(middleData, repository.getNameRepository().fromName(middleName).getData()); String newestName = "newest-name"; + String newestReducedName = "newest-name"; String newestData = "newest-data"; transactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), middleName, newestName, newestData); TransactionUtils.signAndMint(repository, transactionData, alice); @@ -273,6 +356,14 @@ public void testDoubleUpdateData() throws DataException { // Check data is correct assertEquals(newestData, repository.getNameRepository().fromName(newestName).getData()); + // Check initial name no longer exists + assertFalse(repository.getNameRepository().nameExists(initialName)); + assertNull(repository.getNameRepository().fromReducedName(initialReducedName)); + + // Check newest name exists + assertTrue(repository.getNameRepository().nameExists(newestName)); + assertNotNull(repository.getNameRepository().fromReducedName(newestReducedName)); + // orphan and recheck BlockUtils.orphanLastBlock(repository); @@ -284,6 +375,10 @@ public void testDoubleUpdateData() throws DataException { // Check data is correct assertEquals(initialData, repository.getNameRepository().fromName(initialName).getData()); + + // Check initial name exists again + assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); } } @@ -294,38 +389,70 @@ public void testIntermediateUpdateData() throws DataException { // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String initialName = "initial-name"; - String initialData = "initial-data"; + String initialReducedName = "initia1-name"; + String initialData = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); + // Check initial name exists + assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); + // Don't update data, but update name. // This tests whether reverting a future update/sale can find the correct previous data String middleName = "middle-name"; + String middleReducedName = "midd1e-name"; String middleData = ""; transactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), initialName, middleName, middleData); TransactionUtils.signAndMint(repository, transactionData, alice); + // Check original name no longer exists + assertFalse(repository.getNameRepository().nameExists(initialName)); + assertNull(repository.getNameRepository().fromReducedName(initialReducedName)); + + // Check middle name exists + assertTrue(repository.getNameRepository().nameExists(middleName)); + assertNotNull(repository.getNameRepository().fromReducedName(middleReducedName)); + // Check data is correct assertEquals(initialData, repository.getNameRepository().fromName(middleName).getData()); String newestName = "newest-name"; + String newestReducedName = "newest-name"; String newestData = "newest-data"; transactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), middleName, newestName, newestData); TransactionUtils.signAndMint(repository, transactionData, alice); + // Check middle name no longer exists + assertFalse(repository.getNameRepository().nameExists(middleName)); + assertNull(repository.getNameRepository().fromReducedName(middleReducedName)); + + // Check newest name exists + assertTrue(repository.getNameRepository().nameExists(newestName)); + assertNotNull(repository.getNameRepository().fromReducedName(newestReducedName)); + // Check data is correct assertEquals(newestData, repository.getNameRepository().fromName(newestName).getData()); // orphan and recheck BlockUtils.orphanLastBlock(repository); + // Check middle name exists + assertTrue(repository.getNameRepository().nameExists(middleName)); + assertNotNull(repository.getNameRepository().fromReducedName(middleReducedName)); + // Check data is correct assertEquals(initialData, repository.getNameRepository().fromName(middleName).getData()); // orphan and recheck BlockUtils.orphanLastBlock(repository); + // Check initial name exists + assertTrue(repository.getNameRepository().nameExists(initialName)); + assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); + // Check data is correct assertEquals(initialData, repository.getNameRepository().fromName(initialName).getData()); } diff --git a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java new file mode 100644 index 000000000..b1c5ec4f6 --- /dev/null +++ b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java @@ -0,0 +1,114 @@ +package org.qortal.test.network; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.junit.Test; +import org.qortal.data.network.OnlineAccountData; +import org.qortal.network.message.*; +import org.qortal.transform.Transformer; + +import java.nio.ByteBuffer; +import java.security.Security; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class OnlineAccountsTests { + + private static final Random RANDOM = new Random(); + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + } + + + @Test + public void testGetOnlineAccountsV2() throws Message.MessageException { + List onlineAccountsOut = generateOnlineAccounts(false); + + Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut); + + byte[] messageBytes = messageOut.toBytes(); + ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes); + + GetOnlineAccountsV2Message messageIn = (GetOnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer); + + List onlineAccountsIn = messageIn.getOnlineAccounts(); + + assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size()); + assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut)); + + Message oldMessageOut = new GetOnlineAccountsMessage(onlineAccountsOut); + byte[] oldMessageBytes = oldMessageOut.toBytes(); + + long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count(); + + System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d", + onlineAccountsOut.size(), + numTimestamps, + numTimestamps != 1 ? "s" : "", + oldMessageBytes.length, + messageBytes.length)); + } + + @Test + public void testOnlineAccountsV2() throws Message.MessageException { + List onlineAccountsOut = generateOnlineAccounts(true); + + Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut); + + byte[] messageBytes = messageOut.toBytes(); + ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes); + + OnlineAccountsV2Message messageIn = (OnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer); + + List onlineAccountsIn = messageIn.getOnlineAccounts(); + + assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size()); + assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut)); + + Message oldMessageOut = new OnlineAccountsMessage(onlineAccountsOut); + byte[] oldMessageBytes = oldMessageOut.toBytes(); + + long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count(); + + System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d", + onlineAccountsOut.size(), + numTimestamps, + numTimestamps != 1 ? "s" : "", + oldMessageBytes.length, + messageBytes.length)); + } + + private List generateOnlineAccounts(boolean withSignatures) { + List onlineAccounts = new ArrayList<>(); + + int numTimestamps = RANDOM.nextInt(2) + 1; // 1 or 2 + + for (int t = 0; t < numTimestamps; ++t) { + int numAccounts = RANDOM.nextInt(3000); + + for (int a = 0; a < numAccounts; ++a) { + byte[] sig = null; + if (withSignatures) { + sig = new byte[Transformer.SIGNATURE_LENGTH]; + RANDOM.nextBytes(sig); + } + + byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + RANDOM.nextBytes(pubkey); + + onlineAccounts.add(new OnlineAccountData(t << 32, sig, pubkey)); + } + } + + return onlineAccounts; + } + +} diff --git a/src/test/resources/arbitrary/demo1/dir1/dir2/lorem5.txt b/src/test/resources/arbitrary/demo1/dir1/dir2/lorem5.txt new file mode 100644 index 000000000..ef07da1fb --- /dev/null +++ b/src/test/resources/arbitrary/demo1/dir1/dir2/lorem5.txt @@ -0,0 +1 @@ +Pellentesque laoreet laoreet dui ut volutpat. diff --git a/src/test/resources/arbitrary/demo1/dir1/lorem4.txt b/src/test/resources/arbitrary/demo1/dir1/lorem4.txt new file mode 100644 index 000000000..6ac4bdd86 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/dir1/lorem4.txt @@ -0,0 +1,10 @@ +Pellentesque mollis risus laoreet neque lobortis, ut euismod nisl gravida. +Nullam sit amet scelerisque sapien, id aliquet elit. Suspendisse eu +accumsan eros. Nullam non nunc ut risus facilisis posuere sed sed ipsum. +Pellentesque habitant morbi tristique senectus et netus et malesuada fames +ac turpis egestas. Nullam magna felis, vehicula a accumsan luctus, vulputate +vitae justo. Integer mollis lacus eu nisi iaculis, ac ultrices sem aliquam. +Sed ac lacus eget nibh posuere sodales. Phasellus sodales, augue ac +tincidunt scelerisque, mi erat varius mauris, sed blandit ex nisl ut +justo. Etiam ac nisl venenatis, malesuada odio vitae, blandit velit. +Phasellus congue leo a porttitor hendrerit. diff --git a/src/test/resources/arbitrary/demo1/lorem1.txt b/src/test/resources/arbitrary/demo1/lorem1.txt new file mode 100644 index 000000000..5721466c0 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/lorem1.txt @@ -0,0 +1,10 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. +Ut ligula felis, imperdiet nec placerat at, placerat +quis diam. Praesent a ultricies lacus. +Aenean luctus blandit dui. Quisque vel augue +diam. Nulla libero libero, condimentum sed +accumsan eu, elementum sit amet turpis. +In semper risus ac libero lobortis, +ut consectetur urna euismod. +Donec ut erat quis mi eleifend tincidunt +aliquet vitae lacus. diff --git a/src/test/resources/arbitrary/demo1/lorem2.txt b/src/test/resources/arbitrary/demo1/lorem2.txt new file mode 100644 index 000000000..8a9c4367a --- /dev/null +++ b/src/test/resources/arbitrary/demo1/lorem2.txt @@ -0,0 +1 @@ +Quisque viverra neque quis eros dapibus diff --git a/src/test/resources/arbitrary/demo1/lorem3.txt b/src/test/resources/arbitrary/demo1/lorem3.txt new file mode 100644 index 000000000..5db7e9851 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/lorem3.txt @@ -0,0 +1 @@ +Sed ac magna pretium, suscipit mauris sed, ultrices nunc. diff --git a/src/test/resources/arbitrary/demo2/dir1/dir2/lorem5.txt b/src/test/resources/arbitrary/demo2/dir1/dir2/lorem5.txt new file mode 100644 index 000000000..73b058e4c --- /dev/null +++ b/src/test/resources/arbitrary/demo2/dir1/dir2/lorem5.txt @@ -0,0 +1 @@ +Pellentesque laoreet laoreet dui ut volutpat. Sentence added. diff --git a/src/test/resources/arbitrary/demo2/dir1/lorem4.txt b/src/test/resources/arbitrary/demo2/dir1/lorem4.txt new file mode 100644 index 000000000..30c179433 --- /dev/null +++ b/src/test/resources/arbitrary/demo2/dir1/lorem4.txt @@ -0,0 +1,11 @@ +Pellentesque mollis risus laoreet neque lobortis, ut euismod nisl gravida. +Nullam sit amet scelerisque sapien, id aliquet elit. Suspendisse eu +accumsan eros. Nullam non nunc ut risus facilisis posuere sed sed ipsum. +Pellentesque habitant morbi tristique senectus et netus et malesuada fames +ac turpis egestas. Nullam magna felis; vehicula a accumsan luctus, vulputate +vitae justo. Integer mollis lacus eu nisi iaculis, ac ultrices sem aliquam. +Sed ac lacus eget nibh posuere sodales. Phasellus sodales, augue ac +tincidunt scelerisque, mi erat Varius mauris, sed blandit ex nisl ut +justo. Etiam ac nisl venenatis, malesuada odio vitae, blandit velit. +Phasellus congue leo a porttitor hendrerit. +Line added. diff --git a/src/test/resources/arbitrary/demo2/lorem1.txt b/src/test/resources/arbitrary/demo2/lorem1.txt new file mode 100644 index 000000000..4df355531 --- /dev/null +++ b/src/test/resources/arbitrary/demo2/lorem1.txt @@ -0,0 +1,10 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. +Ut ligula felis, imperdiet nec placerat at, placerat +quis diam. Praesent a ultricies lacus. +Aenean luctus blandit dui. Quisque vel augue +diam. Nulla libero libero; condimentum sed +accumsan eu, elementum sit amet turpis. +In semper risus ac libero lobortis, +ut consectetur urna euismod. +Donec ut erat quis mi eleifend tincidunt +aliquet vitae lacus. diff --git a/src/test/resources/arbitrary/demo2/lorem2.txt b/src/test/resources/arbitrary/demo2/lorem2.txt new file mode 100644 index 000000000..de4628f22 --- /dev/null +++ b/src/test/resources/arbitrary/demo2/lorem2.txt @@ -0,0 +1,2 @@ +Quisque viverra neque +quis eros dapibus diff --git a/src/test/resources/arbitrary/demo2/lorem3.txt b/src/test/resources/arbitrary/demo2/lorem3.txt new file mode 100644 index 000000000..5db7e9851 --- /dev/null +++ b/src/test/resources/arbitrary/demo2/lorem3.txt @@ -0,0 +1 @@ +Sed ac magna pretium, suscipit mauris sed, ultrices nunc. diff --git a/src/test/resources/arbitrary/demo3/dir1/dir2/lorem5.txt b/src/test/resources/arbitrary/demo3/dir1/dir2/lorem5.txt new file mode 100644 index 000000000..74d0fda9f --- /dev/null +++ b/src/test/resources/arbitrary/demo3/dir1/dir2/lorem5.txt @@ -0,0 +1 @@ +Pellentesque laoreet laoreet dui ut volutpat. Sentence modified. diff --git a/src/test/resources/arbitrary/demo3/dir1/lorem4.txt b/src/test/resources/arbitrary/demo3/dir1/lorem4.txt new file mode 100644 index 000000000..d2cd1ea4b --- /dev/null +++ b/src/test/resources/arbitrary/demo3/dir1/lorem4.txt @@ -0,0 +1,11 @@ +Pellentesque mollis risus laoreet neque lobortis, ut euismod nisl gravida. +Nullam sit amet scelerisque sapien, id aliquet elit. Suspendisse eu +accumsan eros. Nullam non nunc ut risus facilisis posuere sed sed ipsum. +Pellentesque habitant morbi tristique senectus et netus et malesuada fames +ac turpis egestas. Nullam magna felis; vehicula a accumsan luctus, vulputate +vitae justo. Integer mollis lacus eu nisi iaculis, ac ultrices sem aliquam. +Sed ac lacus eget nibh posuere sodales. Phasellus sodales, augue ac +tincidunt scelerisque, mi erat Varius mauris, sed blandit ex nisl ut +justo. Etiam ac nisl venenatis, malesuada odio vitae, blandit velit. +Phasellus congue leo a porttitor hendrerit. +Line modified. diff --git a/src/test/resources/arbitrary/demo3/lorem1.txt b/src/test/resources/arbitrary/demo3/lorem1.txt new file mode 100644 index 000000000..0def0e23c --- /dev/null +++ b/src/test/resources/arbitrary/demo3/lorem1.txt @@ -0,0 +1,10 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. +Ut ligula felis, imperdiet nec placerat at, placerat +quis diam. Praesent a ultricies lacus. +Aenean luctus blandit dui. Quisque vel augue +diam. Nulla libero libero; condimentum sed +accumsan eu, elementum sit amet turpis. +In semper risus ac libero lobortis, +ut consectetur urna euismod! +Donec ut erat quis mi eleifend tincidunt +aliquet vitae lacus. diff --git a/src/test/resources/arbitrary/demo3/lorem2.txt b/src/test/resources/arbitrary/demo3/lorem2.txt new file mode 100644 index 000000000..47158e176 --- /dev/null +++ b/src/test/resources/arbitrary/demo3/lorem2.txt @@ -0,0 +1,10 @@ +Quisque viverra neque +quis eros dapibus +Quisque viverra neque +quis eros dapibus +Quisque viverra neque +quis eros dapibus +Quisque viverra neque +quis eros dapibus +Quisque viverra neque +quis eros dapibus diff --git a/src/test/resources/arbitrary/demo3/lorem3.txt b/src/test/resources/arbitrary/demo3/lorem3.txt new file mode 100644 index 000000000..5db7e9851 --- /dev/null +++ b/src/test/resources/arbitrary/demo3/lorem3.txt @@ -0,0 +1 @@ +Sed ac magna pretium, suscipit mauris sed, ultrices nunc. diff --git a/src/test/resources/log4j2-test.properties b/src/test/resources/log4j2-test.properties index 5c8723e16..71334cf03 100644 --- a/src/test/resources/log4j2-test.properties +++ b/src/test/resources/log4j2-test.properties @@ -87,7 +87,7 @@ appender.rolling.type = RollingFile appender.rolling.name = FILE appender.rolling.layout.type = PatternLayout appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -appender.rolling.filePattern = ${filename}.%i +appender.rolling.filePattern = ./${filename}.%i appender.rolling.policy.type = SizeBasedTriggeringPolicy appender.rolling.policy.size = 4MB # Set the immediate flush to true (default) diff --git a/src/test/resources/test-chain-v2-founder-rewards.json b/src/test/resources/test-chain-v2-founder-rewards.json index 4ad21f35b..c2a61503e 100644 --- a/src/test/resources/test-chain-v2-founder-rewards.json +++ b/src/test/resources/test-chain-v2-founder-rewards.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, @@ -44,7 +45,12 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-leftover-reward.json b/src/test/resources/test-chain-v2-leftover-reward.json index d402aa955..be04d7a20 100644 --- a/src/test/resources/test-chain-v2-leftover-reward.json +++ b/src/test/resources/test-chain-v2-leftover-reward.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, @@ -44,7 +45,12 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-minting.json b/src/test/resources/test-chain-v2-minting.json index 02b31ef9e..d79c8e980 100644 --- a/src/test/resources/test-chain-v2-minting.json +++ b/src/test/resources/test-chain-v2-minting.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, @@ -44,7 +45,12 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-qora-holder-extremes.json b/src/test/resources/test-chain-v2-qora-holder-extremes.json index 2962f7a70..08c6fab3b 100644 --- a/src/test/resources/test-chain-v2-qora-holder-extremes.json +++ b/src/test/resources/test-chain-v2-qora-holder-extremes.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, @@ -44,7 +45,12 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-qora-holder.json b/src/test/resources/test-chain-v2-qora-holder.json index 11ccb0b03..804087b79 100644 --- a/src/test/resources/test-chain-v2-qora-holder.json +++ b/src/test/resources/test-chain-v2-qora-holder.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, @@ -44,7 +45,12 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-reward-levels.json b/src/test/resources/test-chain-v2-reward-levels.json new file mode 100644 index 000000000..2eae612d6 --- /dev/null +++ b/src/test/resources/test-chain-v2-reward-levels.json @@ -0,0 +1,77 @@ +{ + "isTestChain": true, + "blockTimestampMargin": 500, + "transactionExpiryPeriod": 86400000, + "maxBlockSize": 2097152, + "maxBytesPerUnitFee": 1024, + "unitFee": "0.1", + "nameRegistrationUnitFee": "5", + "requireGroupForApproval": false, + "minAccountLevelToRewardShare": 5, + "maxRewardSharesPerMintingAccount": 20, + "founderEffectiveMintingLevel": 10, + "onlineAccountSignaturesMinLifetime": 3600000, + "onlineAccountSignaturesMaxLifetime": 86400000, + "rewardsByHeight": [ + { "height": 1, "reward": 100 }, + { "height": 11, "reward": 10 }, + { "height": 21, "reward": 1 } + ], + "sharesByLevel": [ + { "levels": [ 1, 2 ], "share": 0.05 }, + { "levels": [ 3, 4 ], "share": 0.10 }, + { "levels": [ 5, 6 ], "share": 0.15 }, + { "levels": [ 7, 8 ], "share": 0.20 }, + { "levels": [ 9, 10 ], "share": 0.25 } + ], + "qoraHoldersShare": 0.20, + "qoraPerQortReward": 250, + "blocksNeededByLevel": [ 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 ], + "blockTimingsByHeight": [ + { "height": 1, "target": 60000, "deviation": 30000, "power": 0.2 } + ], + "ciyamAtSettings": { + "feePerStep": "0.0001", + "maxStepsPerRound": 500, + "stepsPerFunctionCall": 10, + "minutesPerBlock": 1 + }, + "featureTriggers": { + "messageHeight": 0, + "atHeight": 0, + "assetsTimestamp": 0, + "votingTimestamp": 0, + "arbitraryTimestamp": 0, + "powfixTimestamp": 0, + "qortalTimestamp": 0, + "newAssetPricingTimestamp": 0, + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 6, + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 + }, + "genesisInfo": { + "version": 4, + "timestamp": 0, + "transactions": [ + { "type": "ISSUE_ASSET", "assetName": "QORT", "description": "QORT native coin", "data": "", "quantity": 0, "isDivisible": true, "fee": 0 }, + { "type": "ISSUE_ASSET", "assetName": "Legacy-QORA", "description": "Representative legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true }, + { "type": "ISSUE_ASSET", "assetName": "QORT-from-QORA", "description": "QORT gained from holding legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true }, + + { "type": "GENESIS", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "amount": "1000000000" }, + { "type": "GENESIS", "recipient": "QixPbJUwsaHsVEofJdozU9zgVqkK6aYhrK", "amount": "1000000" }, + { "type": "GENESIS", "recipient": "QaUpHNhT3Ygx6avRiKobuLdusppR5biXjL", "amount": "1000000" }, + { "type": "GENESIS", "recipient": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "amount": "1000000" }, + + { "type": "ACCOUNT_FLAGS", "target": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "andMask": -1, "orMask": 1, "xorMask": 0 }, + { "type": "REWARD_SHARE", "minterPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "rewardSharePublicKey": "7PpfnvLSG7y4HPh8hE7KoqAjLCkv7Ui6xw4mKAkbZtox", "sharePercent": 100 }, + + { "type": "ACCOUNT_LEVEL", "target": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "level": 1 }, + { "type": "ACCOUNT_LEVEL", "target": "QixPbJUwsaHsVEofJdozU9zgVqkK6aYhrK", "level": 1 }, + { "type": "ACCOUNT_LEVEL", "target": "QaUpHNhT3Ygx6avRiKobuLdusppR5biXjL", "level": 1 }, + { "type": "ACCOUNT_LEVEL", "target": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "level": 2 } + ] + } +} diff --git a/src/test/resources/test-chain-v2-reward-scaling.json b/src/test/resources/test-chain-v2-reward-scaling.json index e454d8e75..6842a7270 100644 --- a/src/test/resources/test-chain-v2-reward-scaling.json +++ b/src/test/resources/test-chain-v2-reward-scaling.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, @@ -44,7 +45,12 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index 1939f3579..8fc7f9571 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, @@ -44,7 +45,12 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-settings-v2-bitcoin-regtest.json b/src/test/resources/test-settings-v2-bitcoin-regtest.json index d996c9fe6..687c240d6 100644 --- a/src/test/resources/test-settings-v2-bitcoin-regtest.json +++ b/src/test/resources/test-settings-v2-bitcoin-regtest.json @@ -1,8 +1,13 @@ { + "repositoryPath": "testdb", "bitcoinNet": "REGTEST", + "litecoinNet": "REGTEST", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2.json", + "exportPath": "qortal-backup-test", + "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-block-archive.json b/src/test/resources/test-settings-v2-block-archive.json new file mode 100644 index 000000000..c5ed1aa80 --- /dev/null +++ b/src/test/resources/test-settings-v2-block-archive.json @@ -0,0 +1,13 @@ +{ + "bitcoinNet": "TEST3", + "litecoinNet": "TEST3", + "restrictedApi": false, + "blockchainConfig": "src/test/resources/test-chain-v2.json", + "exportPath": "qortal-backup-test", + "bootstrap": false, + "wipeUnconfirmedOnStart": false, + "testNtpOffset": 0, + "minPeers": 0, + "pruneBlockLimit": 100, + "repositoryPath": "dbtest" +} diff --git a/src/test/resources/test-settings-v2-founder-rewards.json b/src/test/resources/test-settings-v2-founder-rewards.json index c89df1878..02d71d761 100644 --- a/src/test/resources/test-settings-v2-founder-rewards.json +++ b/src/test/resources/test-settings-v2-founder-rewards.json @@ -1,7 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-founder-rewards.json", + "exportPath": "qortal-backup-test", + "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-leftover-reward.json b/src/test/resources/test-settings-v2-leftover-reward.json index bdbc1d521..185bbeba5 100644 --- a/src/test/resources/test-settings-v2-leftover-reward.json +++ b/src/test/resources/test-settings-v2-leftover-reward.json @@ -1,7 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-leftover-reward.json", + "exportPath": "qortal-backup-test", + "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-minting.json b/src/test/resources/test-settings-v2-minting.json index 9c72c375d..b56458125 100644 --- a/src/test/resources/test-settings-v2-minting.json +++ b/src/test/resources/test-settings-v2-minting.json @@ -1,7 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-minting.json", + "exportPath": "qortal-backup-test", + "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-qora-holder-extremes.json b/src/test/resources/test-settings-v2-qora-holder-extremes.json index b311fbf27..e20fddf0d 100644 --- a/src/test/resources/test-settings-v2-qora-holder-extremes.json +++ b/src/test/resources/test-settings-v2-qora-holder-extremes.json @@ -1,7 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-qora-holder-extremes.json", + "exportPath": "qortal-backup-test", + "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-qora-holder.json b/src/test/resources/test-settings-v2-qora-holder.json index 83b232873..9d7d25679 100644 --- a/src/test/resources/test-settings-v2-qora-holder.json +++ b/src/test/resources/test-settings-v2-qora-holder.json @@ -1,7 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-qora-holder.json", + "exportPath": "qortal-backup-test", + "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-reward-levels.json b/src/test/resources/test-settings-v2-reward-levels.json new file mode 100644 index 000000000..3ee0179df --- /dev/null +++ b/src/test/resources/test-settings-v2-reward-levels.json @@ -0,0 +1,11 @@ +{ + "repositoryPath": "testdb", + "restrictedApi": false, + "blockchainConfig": "src/test/resources/test-chain-v2-reward-levels.json", + "exportPath": "qortal-backup-test", + "bootstrap": false, + "wipeUnconfirmedOnStart": false, + "testNtpOffset": 0, + "minPeers": 0, + "pruneBlockLimit": 100 +} diff --git a/src/test/resources/test-settings-v2-reward-scaling.json b/src/test/resources/test-settings-v2-reward-scaling.json index 262938b71..fa02ebe72 100644 --- a/src/test/resources/test-settings-v2-reward-scaling.json +++ b/src/test/resources/test-settings-v2-reward-scaling.json @@ -1,7 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-reward-scaling.json", + "exportPath": "qortal-backup-test", + "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2.json b/src/test/resources/test-settings-v2.json index 1cefddeeb..b2ad3db8a 100644 --- a/src/test/resources/test-settings-v2.json +++ b/src/test/resources/test-settings-v2.json @@ -1,8 +1,19 @@ { + "repositoryPath": "testdb", "bitcoinNet": "TEST3", + "litecoinNet": "TEST3", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2.json", + "exportPath": "qortal-backup-test", + "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100, + "bootstrapFilenamePrefix": "test-", + "dataPath": "data-test", + "tempDataPath": "data-test/_temp", + "listsPath": "lists-test", + "storagePolicy": "FOLLOWED_OR_VIEWED", + "maxStorageCapacity": 104857600 } diff --git a/start.sh b/start.sh new file mode 100755 index 000000000..b3db54fea --- /dev/null +++ b/start.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +# There's no need to run as root, so don't allow it, for security reasons +if [ "$USER" = "root" ]; then + echo "Please su to non-root user before running" + exit +fi + +# Validate Java is installed and the minimum version is available +MIN_JAVA_VER='11' + +if command -v java > /dev/null 2>&1; then + # Example: openjdk version "11.0.6" 2020-01-14 + version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | cut -d'.' -f1,2) + if echo "${version}" "${MIN_JAVA_VER}" | awk '{ if ($2 > 0 && $1 >= $2) exit 0; else exit 1}'; then + echo 'Passed Java version check' + else + echo "Please upgrade your Java to version ${MIN_JAVA_VER} or greater" + exit 1 + fi +else + echo "Java is not available, please install Java ${MIN_JAVA_VER} or greater" + exit 1 +fi + +# No qortal.jar but we have a Maven built one? +# Be helpful and copy across to correct location +if [ ! -e qortal.jar -a -f target/qortal*.jar ]; then + echo "Copying Maven-built Qortal JAR to correct pathname" + cp target/qortal*.jar qortal.jar +fi + +# Limits Java JVM stack size and maximum heap usage. +# Comment out for bigger systems, e.g. non-routers +# or when API documentation is enabled +# JVM_MEMORY_ARGS="-Xss256k -Xmx128m" + +# Although java.net.preferIPv4Stack is supposed to be false +# by default in Java 11, on some platforms (e.g. FreeBSD 12), +# it is overridden to be true by default. Hence we explicitly +# set it to false to obtain desired behaviour. +nohup nice -n 20 java \ + -Djava.net.preferIPv4Stack=false \ + ${JVM_MEMORY_ARGS} \ + -jar qortal.jar \ + 1>run.log 2>&1 & + +# Save backgrounded process's PID +echo $! > run.pid +echo qortal running as pid $! diff --git a/stop.sh b/stop.sh index 2f26bc1ff..d79a51db0 100755 --- a/stop.sh +++ b/stop.sh @@ -21,21 +21,60 @@ fi read pid 2>/dev/null /dev/null 2>&1; then - echo "Qortal node responded and should be shutting down" - if [ "${is_pid_valid}" -eq 0 ]; then - echo -n "Monitoring for Qortal node to end" - while s=`ps -p $pid -o stat=` && [[ "$s" && "$s" != 'Z' ]]; do - echo -n . - sleep 1 - done - echo - echo "${green}Qortal ended gracefully${normal}" - rm -f run.pid - fi - exit 0 -else - echo "${red}No response from Qortal node - not running?${normal}" - exit 1 +# Swap out the API port if the --testnet (or -t) argument is specified +api_port=12391 +if [[ "$@" = *"--testnet"* ]] || [[ "$@" = *"-t"* ]]; then + api_port=62391 +fi + +# Attempt to locate the process ID if we don't have one +if [ -z "${pid}" ]; then + pid=$(ps aux | grep '[q]ortal.jar' | head -n 1 | awk '{print $2}') + is_pid_valid=$? fi + +# Locate the API key if it exists +apikey=$(cat apikey.txt) +success=0 + +# Try and stop via the API +if [ -n "$apikey" ]; then + echo "Stopping Qortal via API..." + if curl --url "http://localhost:${api_port}/admin/stop?apiKey=$apikey" 1>/dev/null 2>&1; then + success=1 + fi +fi + +# Try to kill process with SIGTERM +if [ "$success" -ne 1 ] && [ -n "$pid" ]; then + echo "Stopping Qortal process $pid..." + if kill -15 "${pid}"; then + success=1 + fi +fi + +# Warn and exit if still no success +if [ "$success" -ne 1 ]; then + if [ -n "$pid" ]; then + echo "${red}Stop command failed - not running with process id ${pid}?${normal}" + else + echo "${red}Stop command failed - not running?${normal}" + fi + exit 1 +fi + +if [ "$success" -eq 1 ]; then + echo "Qortal node should be shutting down" + if [ "${is_pid_valid}" -eq 0 ]; then + echo -n "Monitoring for Qortal node to end" + while s=`ps -p $pid -o stat=` && [[ "$s" && "$s" != 'Z' ]]; do + echo -n . + sleep 1 + done + echo + echo "${green}Qortal ended gracefully${normal}" + rm -f run.pid + fi +fi + +exit 0 diff --git a/tools/approve-auto-update.sh b/tools/approve-auto-update.sh index 232fc0998..cbfa280dd 100755 --- a/tools/approve-auto-update.sh +++ b/tools/approve-auto-update.sh @@ -7,7 +7,7 @@ fi printf "Searching for auto-update transactions to approve...\n"; -tx=$( curl --silent --url "http://localhost:${port}/arbitrary/search?txGroupId=1&service=1&confirmationStatus=CONFIRMED&limit=1&reverse=true" ); +tx=$( curl --silent --url "http://localhost:${port}/arbitrary/search?txGroupId=1&service=AUTO_UPDATE&confirmationStatus=CONFIRMED&limit=1&reverse=true" ); if fgrep --silent '"approvalStatus":"PENDING"' <<< "${tx}"; then true else diff --git a/tools/block-timings.sh b/tools/block-timings.sh new file mode 100755 index 000000000..88d8d6432 --- /dev/null +++ b/tools/block-timings.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash + +start_height=$1 +count=$2 +target=$3 +deviation=$4 +power=$5 + +if [ -z "${start_height}" ]; then + echo + echo "Error: missing start height." + echo + echo "Usage:" + echo "block-timings.sh [count] [target] [deviation] [power]" + echo + echo "startheight: a block height, preferably within the untrimmed range, to avoid data gaps" + echo "count: the number of blocks to request and analyse after the start height. Default: 100" + echo "target: the target block time in milliseconds. Originates from blockchain.json. Default: 60000" + echo "deviation: the allowed block time deviation in milliseconds. Originates from blockchain.json. Default: 30000" + echo "power: used when transforming key distance to a time offset. Originates from blockchain.json. Default: 0.2" + echo + exit +fi + +count=${count:=100} +target=${target:=60000} +deviation=${deviation:=30000} +power=${power:=0.2} + +finish_height=$((start_height + count - 1)) +height=$start_height + +echo "Settings:" +echo "Target time offset: ${target}" +echo "Deviation: ${deviation}" +echo "Power transform: ${power}" +echo + +function calculate_time_offset { + local key_distance_ratio=$1 + local transformed=$( echo "" | awk "END {print ${key_distance_ratio} ^ ${power}}") + local time_offset=$(echo "${deviation}*2*${transformed}" | bc) + time_offset=${time_offset%.*} + echo $time_offset +} + + +function fetch_and_process_blocks { + + echo "Fetching blocks from height ${start_height} to ${finish_height}..." + echo + + total_time_offset=0 + errors=0 + + while [ "${height}" -le "${finish_height}" ]; do + block_minting_info=$(curl -s "http://localhost:12391/blocks/byheight/${height}/mintinginfo") + error=$(echo "${block_minting_info}" | jq -r .error) + if [ "${error}" != "null" ]; then + echo "Error fetching minting info for block ${height}" + echo + errors=$((errors+1)) + height=$((height+1)) + continue; + fi + + # Parse minting info + minter_level=$(echo "${block_minting_info}" | jq -r .minterLevel) + online_accounts_count=$(echo "${block_minting_info}" | jq -r .onlineAccountsCount) + key_distance_ratio=$(echo "${block_minting_info}" | jq -r .keyDistanceRatio) + time_delta=$(echo "${block_minting_info}" | jq -r .timeDelta) + timestamp=$(echo "${block_minting_info}" | jq -r .timestamp) + + time_offset=$(calculate_time_offset "${key_distance_ratio}") + block_time=$((target-deviation+time_offset)) + + echo "=== BLOCK ${height} ===" + echo "Timestamp: ${timestamp}" + echo "Minter level: ${minter_level}" + echo "Online accounts: ${online_accounts_count}" + echo "Key distance ratio: ${key_distance_ratio}" + echo "Time offset: ${time_offset}" + echo "Block time (real): ${time_delta}" + echo "Block time (calculated): ${block_time}" + + if [ "${time_delta}" -ne "${block_time}" ]; then + echo "WARNING: Block time mismatch. This is to be expected when using custom settings." + fi + echo + + total_time_offset=$((total_time_offset+block_time)) + + height=$((height+1)) + done + + adjusted_count=$((count-errors)) + if [ "${adjusted_count}" -eq 0 ]; then + echo "No blocks were retrieved." + echo + exit; + fi + + mean_time_offset=$((total_time_offset/adjusted_count)) + time_offset_diff=$((mean_time_offset-target)) + + echo "===================" + echo "===== SUMMARY =====" + echo "===================" + echo "Total blocks retrieved: ${adjusted_count}" + echo "Total blocks failed: ${errors}" + echo "Mean time offset: ${mean_time_offset}ms" + echo "Target time offset: ${target}ms" + echo "Difference from target: ${time_offset_diff}ms" + echo + +} + +function estimate_key_distance_ratio_for_level { + local level=$1 + local example_key_distance="0.5" + echo "(${example_key_distance}/${level})" +} + +function estimate_block_timestamps { + min_block_time=9999999 + max_block_time=0 + + echo "===== BLOCK TIME ESTIMATES =====" + + for level in {1..10}; do + example_key_distance_ratio=$(estimate_key_distance_ratio_for_level "${level}") + time_offset=$(calculate_time_offset "${example_key_distance_ratio}") + block_time=$((target-deviation+time_offset)) + + if [ "${block_time}" -gt "${max_block_time}" ]; then + max_block_time=${block_time} + fi + if [ "${block_time}" -lt "${min_block_time}" ]; then + min_block_time=${block_time} + fi + + echo "Level: ${level}, time offset: ${time_offset}, block time: ${block_time}" + done + block_time_range=$((max_block_time-min_block_time)) + echo "Range: ${block_time_range}" + echo +} + +fetch_and_process_blocks +estimate_block_timestamps diff --git a/tools/build-auto-update.sh b/tools/build-auto-update.sh index db651a398..4124f1e26 100755 --- a/tools/build-auto-update.sh +++ b/tools/build-auto-update.sh @@ -13,7 +13,7 @@ fi cd ${git_dir} # Check we are in 'master' branch -branch_name=$( git symbolic-ref -q HEAD ) +branch_name=$( git symbolic-ref -q HEAD || echo ) branch_name=${branch_name##refs/heads/} echo "Current git branch: ${branch_name}" if [ "${branch_name}" != "master" ]; then @@ -78,5 +78,7 @@ git add ${project}.update git commit --message "XORed, auto-update JAR based on commit ${short_hash}" git push --set-upstream origin --force-with-lease ${update_branch} +branch_name=${branch_name-master} + echo "Changing back to '${branch_name}' branch" git checkout --force ${branch_name} diff --git a/tools/build-release.sh b/tools/build-release.sh new file mode 100755 index 000000000..28b289f72 --- /dev/null +++ b/tools/build-release.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash + +# Change this to where AdvancedInstaller outputs built EXE installers +SCRIPT_DIR=$(dirname $(realpath "$0")) +WINDOWS_INSTALLER_DIR="${SCRIPT_DIR}/../WindowsInstaller/Qortal-SetupFiles" + +set -e +shopt -s expand_aliases + +# optional git tag? +if [ $# -ge 1 ]; then + git_tag="$1" + shift +fi + +saved_pwd=$PWD + +alias SHA256='(sha256 -q || sha256sum | cut -d" " -f1) 2>/dev/null' + +function 3hash { + local zip_src=$1 + local md5hash=$(md5 ${zip_src} | awk '{ print $NF }') + local sha1hash=$(shasum ${zip_src} | awk '{ print $1 }') + local sha256hash=$(sha256sum ${zip_src} | awk '{ print $1 }') + echo "\`MD5: ${md5hash}\`" + echo "\`SHA1: ${sha1hash}\`" + echo "\`SHA256: ${sha256hash}\`" +} + +# Check we are within a git repo +git_dir=$( git rev-parse --show-toplevel ) +if [ -z "${git_dir}" ]; then + echo "Cannot determine top-level directory for git repo" + exit 1 +fi + +# Change to git top-level +cd ${git_dir} + +# Check we are in 'master' branch +# branch_name=$( git symbolic-ref -q HEAD ) || echo "Cannot determine branch name" && exit 1 +# branch_name=${branch_name##refs/heads/} +# if [ "${branch_name}" != "master" ]; then + # echo "Unexpected current branch '${branch_name}' - expecting 'master'" + # exit 1 +# fi + +# Determine project name +project=$( perl -n -e 'if (m/(\w+)<.artifactId>/) { print $1; exit }' pom.xml $) +if [ -z "${project}" ]; then + echo "Unable to determine project name from pom.xml?" + exit 1 +fi + +# Extract git tag +if [ -z "${git_tag}" ]; then + git_tag=$( git tag --points-at HEAD ) + if [ -z "${git_tag}" ]; then + echo "Unable to extract git tag" + exit 1 + fi +fi + +# Find origin URL +git_url=$( git remote get-url origin ) +git_url=https://github.com/${git_url##*:} +git_url=${git_url%%.git} + +# Check for EXE +exe=${project}.exe +exe_src="${WINDOWS_INSTALLER_DIR}/${exe}" +if [ ! -r "${exe_src}" ]; then + echo "Cannot find EXE installer at ${exe_src}" + exit +fi + +# Check for ZIP +zip_filename=${project}.zip +zip_src=${saved_pwd}/${zip_filename} +if [ ! -r "${zip_src}" ]; then + echo "Cannot find ZIP at ${zip_src}" + exit +fi + + + +# Changes +cat <<"__CHANGES__" +*Changes in this release:* +* +__CHANGES__ + +# JAR +cat <<__JAR__ + +### [${project}.jar](${git_url}/releases/download/${git_tag}/${project}.jar) + +If built using OpenJDK 11: +__JAR__ +3hash target/${project}*.jar + +# EXE +cat <<__EXE__ + +### [${exe}](${git_url}/releases/download/${git_tag}/${exe}) + +__EXE__ +3hash "${exe_src}" + +# VirusTotal url is SHA256 of github download url: +virustotal_url=$( echo -n "${git_url}/releases/download/${git_tag}/${exe}" | SHA256 ) +cat <<__VIRUSTOTAL__ + +[VirusTotal report for ${exe}](https://www.virustotal.com/gui/url/${virustotal_url}/detection) +__VIRUSTOTAL__ + +# ZIP +cat <<__ZIP__ + +### [${zip_filename}](${git_url}/releases/download/${git_tag}/${zip_filename}) + +Contains bare minimum of: +* built \`${project}.jar\` +* \`log4j2.properties\` from git repo +* \`start.sh\` from git repo +* \`stop.sh\` from git repo +* \`printf "{\n}\n" > settings.json\` + +All timestamps set to same date-time as commit, obtained via \`git show --no-patch --format=%cI\` +Packed with \`7z a -r -tzip ${zip_filename} ${project}/\` + +__ZIP__ +3hash ${zip_src} diff --git a/tools/build-zip.sh b/tools/build-zip.sh index b5ddea623..b52b5da73 100755 --- a/tools/build-zip.sh +++ b/tools/build-zip.sh @@ -2,6 +2,12 @@ set -e +# Optional git tag? +if [ $# -ge 1 ]; then + git_tag="$1" + shift +fi + saved_pwd=$PWD # Check we are within a git repo @@ -15,13 +21,13 @@ fi cd ${git_dir} # Check we are in 'master' branch -branch_name=$( git symbolic-ref -q HEAD ) -branch_name=${branch_name##refs/heads/} -echo "Current git branch: ${branch_name}" -if [ "${branch_name}" != "master" ]; then - echo "Unexpected current branch '${branch_name}' - expecting 'master'" - exit 1 -fi +# branch_name=$( git symbolic-ref -q HEAD ) || echo "Cannot determine branch name" && exit 1 +# branch_name=${branch_name##refs/heads/} +# echo "Current git branch: ${branch_name}" +# if [ "${branch_name}" != "master" ]; then +# echo "Unexpected current branch '${branch_name}' - expecting 'master'" +# exit 1 +# fi # Determine project name project=$( perl -n -e 'if (m/(\w+)<.artifactId>/) { print $1; exit }' pom.xml $) @@ -31,10 +37,12 @@ if [ -z "${project}" ]; then fi # Extract git tag -git_tag=$( git tag --points-at HEAD ) if [ -z "${git_tag}" ]; then - echo "Unable to extract git tag" - exit 1 + git_tag=$( git tag --points-at HEAD ) + if [ -z "${git_tag}" ]; then + echo "Unable to extract git tag" + exit 1 + fi fi build_dir=/tmp/${project} @@ -47,11 +55,12 @@ cp target/${project}*.jar ${build_dir}/${project}.jar git show HEAD:log4j2.properties > ${build_dir}/log4j2.properties -git show HEAD:run.sh > ${build_dir}/run.sh +git show HEAD:start.sh > ${build_dir}/start.sh +git show HEAD:stop.sh > ${build_dir}/stop.sh printf "{\n}\n" > ${build_dir}/settings.json -touch -d ${commit_ts%%+??:??} ${build_dir} ${build_dir}/* +gtouch -d ${commit_ts%%+??:??} ${build_dir} ${build_dir}/* rm -f ${saved_pwd}/${project}.zip -(cd ${build_dir}/..; 7z a -r -tzip ${saved_pwd}/${project}-${git_tag#v}.zip ${project}/) +(cd ${build_dir}/..; 7z a -r -tzip ${saved_pwd}/${project}.zip ${project}/) diff --git a/tools/peer-heights b/tools/peer-heights new file mode 100755 index 000000000..4ebc21dbc --- /dev/null +++ b/tools/peer-heights @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Requires: 'qort' script in PATH, and 'jq' utility installed + +set -e + +# Any extra args passed to us are also passed to 'qort', just prior to '-p peers' +qort $@ -p peers | \ + jq -r 'def lpad($len): + tostring | ($len - length) as $l | (" " * $l)[:$l] + .; + .[] | + select(has("lastHeight")) | + "\(.address | lpad(22)) (\(.version)), height \(.lastHeight), sig: \(.lastBlockSignature[0:8]), ts \(.lastBlockTimestamp / 1e3 | strftime("%Y-%m-%d %H:%M:%S"))"' diff --git a/tools/publish-auto-update-v5.pl b/tools/publish-auto-update-v5.pl new file mode 100755 index 000000000..aad49d4e0 --- /dev/null +++ b/tools/publish-auto-update-v5.pl @@ -0,0 +1,160 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use POSIX; +use Getopt::Std; + +sub usage() { + die("usage: $0 [-p api-port] dev-private-key [short-commit-hash]\n"); +} + +my %opt; +getopts('p:', \%opt); + +usage() if @ARGV < 1 || @ARGV > 2; + +my $port = $opt{p} || 12391; +my $privkey = shift @ARGV; +my $commit_hash = shift @ARGV; + +my $git_dir = `git rev-parse --show-toplevel`; +die("Cannot determine git top level dir\n") unless $git_dir; + +chomp $git_dir; +chdir($git_dir) || die("Can't change directory to $git_dir: $!\n"); + +open(POM, '<', 'pom.xml') || die ("Can't open 'pom.xml': $!\n"); +my $project; +while () { + if (m/(\w+)<.artifactId>/o) { + $project = $1; + last; + } +} +close(POM); + +# Do we need to determine commit hash? +unless ($commit_hash) { + # determine git branch + my $branch_name = ` git symbolic-ref -q HEAD `; + chomp $branch_name; + $branch_name =~ s|^refs/heads/||; # ${branch_name##refs/heads/} + + # short-form commit hash on base branch (non-auto-update) + $commit_hash ||= `git show --no-patch --format=%h`; + die("Can't find commit hash\n") if ! defined $commit_hash; + chomp $commit_hash; + printf "Commit hash on '%s' branch: %s\n", $branch_name, $commit_hash; +} else { + printf "Using given commit hash: %s\n", $commit_hash; +} + +# build timestamp / commit timestamp on base branch +my $timestamp = `git show --no-patch --format=%ct ${commit_hash}`; +die("Can't determine commit timestamp\n") if ! defined $timestamp; +$timestamp *= 1000; # Convert to milliseconds + +# locate sha256 utility +my $SHA256 = `which sha256sum || which sha256`; +chomp $SHA256; +die("Can't find sha256sum or sha256\n") unless length($SHA256) > 0; + +# SHA256 of actual update file +my $sha256 = `git show auto-update-${commit_hash}:${project}.update | ${SHA256} | head -c 64`; +die("Can't calculate SHA256 of ${project}.update\n") unless $sha256 =~ m/(\S{64})/; +chomp $sha256; + +# long-form commit hash of HEAD on auto-update branch +my $update_hash = `git rev-parse refs/heads/auto-update-${commit_hash}`; +die("Can't find commit hash for HEAD on auto-update-${commit_hash} branch\n") if ! defined $update_hash; +chomp $update_hash; + +printf "Build timestamp (ms): %d / 0x%016x\n", $timestamp, $timestamp; +printf "Auto-update commit hash: %s\n", $update_hash; +printf "SHA256 of ${project}.update: %s\n", $sha256; + +my $tx_type = 10; +my $tx_timestamp = time() * 1000; +my $tx_group_id = 1; +my $service = 1; +printf "\nARBITRARY(%d) transaction with timestamp %d, txGroupID %d and service %d\n", $tx_type, $tx_timestamp, $tx_group_id, $service; + +my $data_hex = sprintf "%016x%s%s", $timestamp, $update_hash, $sha256; +printf "\nARBITRARY transaction data payload: %s\n", $data_hex; + +my $n_payments = 0; +my $data_type = 1; # RAW_DATA +my $data_length = length($data_hex) / 2; # two hex chars per byte +my $fee = 0; +my $nonce = 0; +my $name_length = 0; +my $identifier_length = 0; +my $method = 0; # PUT +my $secret_length = 0; +my $compression = 0; # None +my $metadata_hash_length = 0; + +die("Something's wrong: data length is not 60 bytes!\n") if $data_length != 60; + +my $pubkey = `curl --silent --url http://localhost:${port}/utils/publickey --data ${privkey}`; +die("Can't convert private key to public key:\n$pubkey\n") unless $pubkey =~ m/^\w{44}$/; +printf "\nPublic key: %s\n", $pubkey; + +my $pubkey_hex = `curl --silent --url http://localhost:${port}/utils/frombase58 --data ${pubkey}`; +die("Can't convert base58 public key to hex:\n$pubkey_hex\n") unless $pubkey_hex =~ m/^[A-Za-z0-9]{64}$/; +printf "Public key hex: %s\n", $pubkey_hex; + +my $address = `curl --silent --url http://localhost:${port}/addresses/convert/${pubkey}`; +die("Can't convert base58 public key to address:\n$address\n") unless $address =~ m/^\w{33,34}$/; +printf "Address: %s\n", $address; + +my $reference = `curl --silent --url http://localhost:${port}/addresses/lastreference/${address}`; +die("Can't fetch last reference for $address:\n$reference\n") unless $reference =~ m/^\w{87,88}$/; +printf "Last reference: %s\n", $reference; + +my $reference_hex = `curl --silent --url http://localhost:${port}/utils/frombase58 --data ${reference}`; +die("Can't convert base58 reference to hex:\n$reference_hex\n") unless $reference_hex =~ m/^[A-Za-z0-9]{128}$/; +printf "Last reference hex: %s\n", $reference_hex; + +my $raw_tx_hex = sprintf("%08x%016x%08x%s%s%08x%08x%08x%08x%08x%08x%08x%08x%02x%08x%s%08x%08x%016x", $tx_type, $tx_timestamp, $tx_group_id, $reference_hex, $pubkey_hex, $nonce, $name_length, $identifier_length, $method, $secret_length, $compression, $n_payments, $service, $data_type, $data_length, $data_hex, $data_length, $metadata_hash_length, $fee); +printf "\nRaw transaction hex:\n%s\n", $raw_tx_hex; + +my $raw_tx = `curl --silent --url http://localhost:${port}/utils/tobase58/${raw_tx_hex}`; +die("Can't convert raw transaction hex to base58:\n$raw_tx\n") unless $raw_tx =~ m/^\w{300,320}$/; # Roughly 305 to 320 base58 chars +printf "\nRaw transaction (base58):\n%s\n", $raw_tx; + +my $computed_tx = `curl --silent -X POST --url http://localhost:${port}/arbitrary/compute -d "${raw_tx}"`; +die("Can't compute nonce for transaction:\n$computed_tx\n") unless $computed_tx =~ m/^\w{300,320}$/; # Roughly 300 to 320 base58 chars +printf "\nRaw computed transaction (base58):\n%s\n", $computed_tx; + +my $sign_data = qq|' { "privateKey": "${privkey}", "transactionBytes": "${computed_tx}" } '|; +my $signed_tx = `curl --silent -H "accept: text/plain" -H "Content-Type: application/json" --url http://localhost:${port}/transactions/sign --data ${sign_data}`; +die("Can't sign raw transaction:\n$signed_tx\n") unless $signed_tx =~ m/^\w{390,410}$/; # +90ish longer than $raw_tx +printf "\nSigned transaction:\n%s\n", $signed_tx; + +# Check we can actually fetch update +my $origin = `git remote get-url origin`; +die("Unable to get github url for 'origin'?\n") unless $origin && $origin =~ m/:(.*)\.git$/; +my $repo = $1; +my $update_url = "https://github.com/${repo}/raw/${update_hash}/${project}.update"; + +my $fetch_result = `curl --silent -o /dev/null --location --range 0-1 --head --write-out '%{http_code}' --url ${update_url}`; +die("\nUnable to fetch update from ${update_url}\n") if $fetch_result ne '200'; +printf "\nUpdate fetchable from ${update_url}\n"; + +# Flush STDOUT after every output +$| = 1; +print "\n"; +for (my $delay = 5; $delay > 0; --$delay) { + printf "\rSubmitting transaction in %d second%s... CTRL-C to abort ", $delay, ($delay != 1 ? 's' : ''); + sleep 1; +} + +printf "\rSubmitting transaction NOW... \n"; +my $result = `curl --silent --url http://localhost:${port}/transactions/process --data ${signed_tx}`; +chomp $result; +die("Transaction wasn't accepted:\n$result\n") unless $result eq 'true'; + +my $decoded_tx = `curl --silent -H "Content-Type: application/json" --url http://localhost:${port}/transactions/decode --data ${signed_tx}`; +printf "\nTransaction accepted:\n$decoded_tx\n"; diff --git a/tools/publish-auto-update.pl b/tools/publish-auto-update.pl index 3493f9647..ad43b2f44 100755 --- a/tools/publish-auto-update.pl +++ b/tools/publish-auto-update.pl @@ -57,9 +57,11 @@ () # locate sha256 utility my $SHA256 = `which sha256sum || which sha256`; +chomp $SHA256; +die("Can't find sha256sum or sha256\n") unless length($SHA256) > 0; # SHA256 of actual update file -my $sha256 = `git show auto-update-${commit_hash}:${project}.update | ${SHA256}`; +my $sha256 = `git show auto-update-${commit_hash}:${project}.update | ${SHA256} | head -c 64`; die("Can't calculate SHA256 of ${project}.update\n") unless $sha256 =~ m/(\S{64})/; chomp $sha256; diff --git a/tools/qdata b/tools/qdata new file mode 100755 index 000000000..5ca61e459 --- /dev/null +++ b/tools/qdata @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +# Qortal defaults +host="localhost" +port=12393 + +if [ -z "$*" ]; then + echo "Usage:" + echo + echo "Host/update data:" + echo "qdata POST [service] [name] PATH [dirpath] " + echo "qdata POST [service] [name] STRING [data-string] " + echo + echo "Fetch data:" + echo "qdata GET [service] [name] " + echo + echo "Notes:" + echo "- When requesting a resource, please use 'default' to indicate a file with no identifier." + echo "- The same applies when specifying the relative path to a file within the data structure; use 'default'" + echo " to indicate a single file resource." + echo + exit +fi + +script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +if [ -f "apikey.txt" ]; then + apikey=$(cat "apikey.txt") +elif [ -f "${script_dir}/../apikey.txt" ]; then + apikey=$(cat "${script_dir}/../apikey.txt") +elif [ -f "${HOME}/qortal/apikey.txt" ]; then + apikey=$(cat "${HOME}/qortal/apikey.txt") +fi + +method=$1 +service=$2 +name=$3 + +if [ -z "${method}" ]; then + echo "Error: missing method"; exit +fi +if [ -z "${service}" ]; then + echo "Error: missing service"; exit +fi +if [ -z "${name}" ]; then + echo "Error: missing name"; exit +fi + + +if [[ "${method}" == "POST" ]]; then + type=$4 + data=$5 + identifier=$6 + + if [ -z "${data}" ]; then + if [[ "${type}" == "PATH" ]]; then + echo "Error: missing directory"; exit + elif [[ "${type}" == "STRING" ]]; then + echo "Error: missing data string"; exit + else + echo "Error: unrecognized type"; exit + fi + fi + if [ -z "${QORTAL_PRIVKEY}" ]; then + echo "Error: missing private key. Set it by running: export QORTAL_PRIVKEY=privkeyhere"; exit + fi + + if [ -z "${identifier}" ]; then + identifier="default" + fi + + # Create type component in URL + if [[ "${type}" == "PATH" ]]; then + type_component="" + elif [[ "${type}" == "STRING" ]]; then + type_component="/string" + fi + + echo "Creating transaction - this can take a while..." + tx_data=$(curl --silent --insecure -X ${method} "http://${host}:${port}/arbitrary/${service}/${name}/${identifier}${type_component}" -H "X-API-KEY: ${apikey}" -d "${data}") + + if [[ "${tx_data}" == *"error"* || "${tx_data}" == *"ERROR"* ]]; then + echo "${tx_data}"; exit + elif [ -z "${tx_data}" ]; then + echo "Error: no transaction data returned"; exit + fi + + echo "Computing nonce..." + computed_tx_data=$(curl --silent --insecure -X POST "http://${host}:${port}/arbitrary/compute" -H "Content-Type: application/json" -H "X-API-KEY: ${apikey}" -d "${tx_data}") + if [[ "${computed_tx_data}" == *"error"* || "${computed_tx_data}" == *"ERROR"* ]]; then + echo "${computed_tx_data}"; exit + fi + + echo "Signing..." + signed_tx_data=$(curl --silent --insecure -X POST "http://${host}:${port}/transactions/sign" -H "Content-Type: application/json" -d "{\"privateKey\":\"${QORTAL_PRIVKEY}\",\"transactionBytes\":\"${computed_tx_data}\"}") + if [[ "${signed_tx_data}" == *"error"* || "${signed_tx_data}" == *"ERROR"* ]]; then + echo "${signed_tx_data}"; exit + fi + + echo "Broadcasting..." + success=$(curl --silent --insecure -X POST "http://${host}:${port}/transactions/process" -H "Content-Type: text/plain" -d "${signed_tx_data}") + if [[ "${success}" == "true" ]]; then + echo "Transaction broadcast successfully" + else + echo "Error when broadcasting transaction. Please try again." + echo "Response: ${success}" + fi + +elif [[ "${method}" == "GET" ]]; then + identifier=$4 + filepath=$5 + rebuild=$6 + + if [ -z "${rebuild}" ]; then + rebuild="false" + fi + + # Handle default + if [[ "${filepath}" == "default" ]]; then + filepath="" + fi + + # We use a different API depending on whether or not an identifier is supplied + if [ -n "${identifier}" ]; then + response=$(curl --silent --insecure -X GET "http://${host}:${port}/arbitrary/${service}/${name}/${identifier}?rebuild=${rebuild}&filepath=${filepath}" -H "X-API-KEY: ${apikey}") + else + response=$(curl --silent --insecure -X GET "http://${host}:${port}/arbitrary/${service}/${name}?rebuild=${rebuild}&filepath=${filepath}" -H "X-API-KEY: ${apikey}") + fi + + if [ -z "${response}" ]; then + echo "Empty response from ${host}:${port}" + fi + if [[ "${response}" == *"error"* || "${response}" == *"ERROR"* ]]; then + echo "${response}"; exit + fi + + echo "${response}" + +fi diff --git a/tools/qort b/tools/qort new file mode 100755 index 000000000..79712b1b5 --- /dev/null +++ b/tools/qort @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# default output post-processor +postproc=cat + +# Qortal defaults +port=12391 +example_host=node10.qortal.org + +# called-as name +name="${0##*/}" + +script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +while [ -n "$*" ]; do + case $1 in + -p) + shift + postproc="json_pp -f json -t json -json_opt utf8,pretty" + ;; + + [Dd][Ee][Ll][Ee][Tt][Ee]) + shift + method="-X DELETE" + ;; + + -l) + shift + src="--interface localhost" + ;; + + -t) + shift + testnet=true + ;; + + -j) + shift + content_type="Content-Type: application/json" + ;; + + *) + break + ;; + esac +done + +if [ "${name}" = "qort" ]; then + port=${testnet:+62391} + port=${port:-12391} + example_host=node10.qortal.org +fi + +if [ -z "$*" ]; then + echo "usage: $name [-l] [-p] [-t] [DELETE] []" + echo "-l: use localhost as source address" + echo "-p: pretty-print JSON output" + echo "-t: use testnet port" + echo "example (using localhost:${port}): $name -p blocks/last" + echo "example: $name -p http://${example_host}:${port}/blocks/last" + echo "example: BASE_URL=http://${example_host}:${port} $name -p blocks/last" + echo "example: BASE_URL=${example_host} $name -p blocks/last" + echo "example: $name -l DELETE peers/known" + exit +fi + +url=$1 +shift + +if [ -f "apikey.txt" ]; then + apikey=$(cat "apikey.txt") +elif [ -f "${script_dir}/../apikey.txt" ]; then + apikey=$(cat "${script_dir}/../apikey.txt") +elif [ -f "${HOME}/qortal/apikey.txt" ]; then + apikey=$(cat "${HOME}/qortal/apikey.txt") +fi + +if [ "${url:0:4}" != "http" ]; then + base_url=${BASE_URL-localhost:${port}} + + if [ "${base_url:0:4}" != "http" ]; then + base_url="http://${base_url}" + fi + + if [ -n "${base_url/#*:[0-9[0-9]*}" ]; then + base_url="${base_url%%/}:${port}" + fi + + url="${base_url%%/}/${url#/}" +fi + +if [ "$#" != 0 ]; then + data="--data" +fi + +curl --silent --insecure --connect-timeout 5 -H "X-API-KEY: ${apikey}" ${content_type:+--header} "${content_type}" ${method} ${src} --url ${url} ${data} "$@" | ${postproc} +echo diff --git a/tools/tx.pl b/tools/tx.pl new file mode 100755 index 000000000..db6958e2a --- /dev/null +++ b/tools/tx.pl @@ -0,0 +1,353 @@ +#!/usr/bin/env perl + +use JSON; +use warnings; +use strict; + +use Getopt::Std; +use File::Basename; + +our %opt; +getopts('dpst', \%opt); + +my $proc = basename($0); + +if (@ARGV < 1) { + print STDERR "usage: $proc [-d] [-p] [-s] [-t] []\n"; + print STDERR "-d: debug, -p: process (broadcast) transaction, -s: sign, -t: testnet\n"; + print STDERR "example: $proc PAYMENT P22kW91AJfDNBj32nVii292hhfo5AgvUYPz5W12ExsjE QxxQZiK7LZBjmpGjRz1FAZSx9MJDCoaHqz 0.1\n"; + print STDERR "example: $proc JOIN_GROUP X92h3hf9k20kBj32nVnoh3XT14o5AgvUYPz5W12ExsjE 3\n"; + print STDERR "example: BASE_URL=node10.qortal.org $proc JOIN_GROUP CB2DW91AJfd47432nVnoh3XT14o5AgvUYPz5W12ExsjE 3\n"; + print STDERR "example: $proc -p sign C4ifh827ffDNBj32nVnoh3XT14o5AgvUYPz5W12ExsjE 111jivxUwerRw...Fjtu\n"; + print STDERR "for help: $proc all\n"; + print STDERR "for help: $proc REGISTER_NAME\n"; + exit 2; +} + +our $BASE_URL = $ENV{BASE_URL} || $opt{t} ? 'http://localhost:62391' : 'http://localhost:12391'; +our $DEFAULT_FEE = 0.001; + +our %TRANSACTION_TYPES = ( + payment => { + url => 'payments/pay', + required => [qw(recipient amount)], + key_name => 'senderPublicKey', + }, + # groups + set_group => { + url => 'groups/setdefault', + required => [qw(defaultGroupId)], + key_name => 'creatorPublicKey', + }, + create_group => { + url => 'groups/create', + required => [qw(groupName description isOpen approvalThreshold)], + key_name => 'creatorPublicKey', + }, + update_group => { + url => 'groups/update', + required => [qw(groupId newOwner newDescription newIsOpen newApprovalThreshold)], + key_name => 'ownerPublicKey', + }, + join_group => { + url => 'groups/join', + required => [qw(groupId)], + key_name => 'joinerPublicKey', + }, + leave_group => { + url => 'groups/leave', + required => [qw(groupId)], + key_name => 'leaverPublicKey', + }, + group_invite => { + url => 'groups/invite', + required => [qw(groupId invitee)], + key_name => 'adminPublicKey', + }, + group_kick => { + url => 'groups/kick', + required => [qw(groupId member reason)], + key_name => 'adminPublicKey', + }, + add_group_admin => { + url => 'groups/addadmin', + required => [qw(groupId member)], + key_name => 'ownerPublicKey', + }, + group_approval => { + url => 'groups/approval', + required => [qw(pendingSignature approval)], + key_name => 'adminPublicKey', + }, + # assets + issue_asset => { + url => 'assets/issue', + required => [qw(assetName description quantity isDivisible)], + key_name => 'issuerPublicKey', + }, + update_asset => { + url => 'assets/update', + required => [qw(assetId newOwner)], + key_name => 'ownerPublicKey', + }, + transfer_asset => { + url => 'assets/transfer', + required => [qw(recipient amount assetId)], + key_name => 'senderPublicKey', + }, + create_order => { + url => 'assets/order', + required => [qw(haveAssetId wantAssetId amount price)], + key_name => 'creatorPublicKey', + }, + # names + register_name => { + url => 'names/register', + required => [qw(name data)], + key_name => 'registrantPublicKey', + }, + update_name => { + url => 'names/update', + required => [qw(newName newData)], + key_name => 'ownerPublicKey', + }, + # reward-shares + reward_share => { + url => 'addresses/rewardshare', + required => [qw(recipient rewardSharePublicKey sharePercent)], + key_name => 'minterPublicKey', + }, + # arbitrary + arbitrary => { + url => 'arbitrary', + required => [qw(service dataType data)], + key_name => 'senderPublicKey', + }, + # chat + chat => { + url => 'chat', + required => [qw(data)], + optional => [qw(recipient isText isEncrypted)], + key_name => 'senderPublicKey', + defaults => { isText => 'true' }, + pow_url => 'chat/compute', + }, + # misc + publicize => { + url => 'addresses/publicize', + required => [], + key_name => 'senderPublicKey', + pow_url => 'addresses/publicize/compute', + }, + # Cross-chain trading + build_trade => { + url => 'crosschain/build', + required => [qw(initialQortAmount finalQortAmount fundingQortAmount secretHash bitcoinAmount)], + optional => [qw(tradeTimeout)], + key_name => 'creatorPublicKey', + defaults => { tradeTimeout => 10800 }, + }, + trade_recipient => { + url => 'crosschain/tradeoffer/recipient', + required => [qw(atAddress recipient)], + key_name => 'creatorPublicKey', + remove => [qw(timestamp reference fee)], + }, + trade_secret => { + url => 'crosschain/tradeoffer/secret', + required => [qw(atAddress secret)], + key_name => 'recipientPublicKey', + remove => [qw(timestamp reference fee)], + }, + # These are fake transaction types to provide utility functions: + sign => { + url => 'transactions/sign', + required => [qw{transactionBytes}], + }, +); + +my $tx_type = lc(shift(@ARGV)); + +if ($tx_type eq 'all') { + printf STDERR "Transaction types: %s\n", join(', ', sort { $a cmp $b } keys %TRANSACTION_TYPES); + exit 2; +} + +my $tx_info = $TRANSACTION_TYPES{$tx_type}; + +if (!$tx_info) { + printf STDERR "Transaction type '%s' unknown\n", uc($tx_type); + exit 1; +} + +my @required = @{$tx_info->{required}}; + +if (@ARGV < @required + 1) { + printf STDERR "usage: %s %s %s", $proc, uc($tx_type), join(' ', map { "<$_>"} @required); + printf STDERR " %s", join(' ', map { "[$_ <$_>]" } @{$tx_info->{optional}}) if exists $tx_info->{optional}; + print "\n"; + exit 2; +} + +my $priv_key = shift @ARGV; + +my $account = account($priv_key); +my $raw; + +if ($tx_type ne 'sign') { + my %extras; + + foreach my $required_arg (@required) { + $extras{$required_arg} = shift @ARGV; + } + + # For CHAT we use a random reference + if ($tx_type eq 'chat') { + $extras{reference} = api('utils/random?length=64'); + } + + %extras = (%extras, %{$tx_info->{defaults}}) if exists $tx_info->{defaults}; + + %extras = (%extras, @ARGV); + + $raw = build_raw($tx_type, $account, %extras); + printf "Raw: %s\n", $raw if $opt{d} || (!$opt{s} && !$opt{p}); + + # Some transaction types require proof-of-work, e.g. CHAT + if (exists $tx_info->{pow_url}) { + $raw = api($tx_info->{pow_url}, $raw); + printf "Raw with PoW: %s\n", $raw if $opt{d}; + } +} else { + $raw = shift @ARGV; + $opt{s}++; +} + +if ($opt{s}) { + my $signed = sign($account->{private}, $raw); + printf "Signed: %s\n", $signed if $opt{d} || $tx_type eq 'sign'; + + if ($opt{p}) { + my $processed = process($signed); + printf "Processed: %s\n", $processed if $opt{d}; + } + + my $hex = api('utils/frombase58', $signed); + # sig is last 64 bytes / 128 chars + my $sighex = substr($hex, -128); + + my $sig58 = api('utils/tobase58/{hex}', '', '{hex}', $sighex); + printf "Signature: %s\n", $sig58; +} + +sub account { + my ($creator) = @_; + + my $account = { private => $creator }; + $account->{public} = api('utils/publickey', $creator); + $account->{address} = api('addresses/convert/{publickey}', '', '{publickey}', $account->{public}); + + return $account; +} + +sub build_raw { + my ($type, $account, %extras) = @_; + + my $tx_info = $TRANSACTION_TYPES{$type}; + die("unknown tx type: $type\n") unless defined $tx_info; + + my $ref = exists $extras{reference} ? $extras{reference} : lastref($account->{address}); + + my %json = ( + timestamp => time * 1000, + reference => $ref, + fee => $DEFAULT_FEE, + ); + + $json{$tx_info->{key_name}} = $account->{public} if exists $tx_info->{key_name}; + + foreach my $required (@{$tx_info->{required}}) { + die("missing tx field: $required\n") unless exists $extras{$required}; + } + + while (my ($key, $value) = each %extras) { + $json{$key} = $value; + } + + if (exists $tx_info->{remove}) { + foreach my $key (@{$tx_info->{remove}}) { + delete $json{$key}; + } + } + + my $json = "{\n"; + while (my ($key, $value) = each %json) { + if (ref($value) eq 'ARRAY') { + $json .= "\t\"$key\": [],\n"; + } else { + $json .= "\t\"$key\": \"$value\",\n"; + } + } + # remove final comma + substr($json, -2, 1) = ''; + $json .= "}\n"; + + printf "%s:\n%s\n", $type, $json if $opt{d}; + + my $raw = api($tx_info->{url}, $json); + return $raw; +} + +sub sign { + my ($private, $raw) = @_; + + my $json = <<" __JSON__"; + { + "privateKey": "$private", + "transactionBytes": "$raw" + } + __JSON__ + + return api('transactions/sign', $json); +} + +sub process { + my ($signed) = @_; + + return api('transactions/process', $signed); +} + +sub lastref { + my ($address) = @_; + + return api('addresses/lastreference/{address}', '', '{address}', $address) +} + +sub api { + my ($endpoint, $postdata, @args) = @_; + + my $url = $endpoint; + my $method = 'GET'; + + for (my $i = 0; $i < @args; $i += 2) { + my $placemarker = $args[$i]; + my $value = $args[$i + 1]; + + $url =~ s/$placemarker/$value/g; + } + + my $curl = "curl --silent --output - --url '$BASE_URL/$url'"; + if (defined $postdata && $postdata ne '') { + $postdata =~ tr|\n| |s; + $curl .= " --header 'Content-Type: application/json' --data-binary '$postdata'"; + $method = 'POST'; + } + my $response = `$curl 2>/dev/null`; + chomp $response; + + if ($response eq '' || substr($response, 0, 6) eq '' || $response =~ m/(^\{|,)"error":(\d+)[,}]/) { + die("API call '$method $BASE_URL/$endpoint' failed:\n$response\n"); + } + + return $response; +}