Skip to content

Commit be1749c

Browse files
authored
fix calculate checkSum when using Java9IntHash (#4140)
* fix calculate checkSum when using Java9IntHash * - * improve performence * add test * edit test * rename test * fix license * address comments
1 parent 11ccebb commit be1749c

2 files changed

Lines changed: 132 additions & 1 deletion

File tree

circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package com.scurrilous.circe.checksum;
2020

2121
import io.netty.buffer.ByteBuf;
22+
import io.netty.buffer.CompositeByteBuf;
2223
import io.netty.util.concurrent.FastThreadLocal;
2324
import java.lang.reflect.InvocationTargetException;
2425
import java.lang.reflect.Method;
@@ -106,14 +107,23 @@ public int resume(int current, ByteBuf buffer, int offset, int len) {
106107
} else if (buffer.hasArray()) {
107108
int arrayOffset = buffer.arrayOffset() + offset;
108109
negCrc = resume(negCrc, buffer.array(), arrayOffset, len);
110+
} else if (buffer instanceof CompositeByteBuf) {
111+
CompositeByteBuf compositeByteBuf = (CompositeByteBuf) buffer;
112+
int loopedCurrent = current;
113+
for (int i = 0; i < compositeByteBuf.numComponents(); i ++) {
114+
loopedCurrent = resume(loopedCurrent, compositeByteBuf.component(i));
115+
}
116+
return loopedCurrent;
109117
} else {
110118
byte[] b = TL_BUFFER.get();
111119
int toRead = len;
120+
int loopOffset = offset;
112121
while (toRead > 0) {
113122
int length = Math.min(toRead, b.length);
114-
buffer.slice(offset, len).readBytes(b, 0, length);
123+
buffer.slice(loopOffset, length).readBytes(b, 0, length);
115124
negCrc = resume(negCrc, b, 0, length);
116125
toRead -= length;
126+
loopOffset += length;
117127
}
118128
}
119129

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package com.scurrilous.circe.checksum;
20+
21+
import io.netty.buffer.ByteBuf;
22+
import io.netty.buffer.ByteBufAllocator;
23+
import io.netty.buffer.CompositeByteBuf;
24+
import io.netty.buffer.DuplicatedByteBuf;
25+
import java.util.Random;
26+
import lombok.extern.slf4j.Slf4j;
27+
import org.junit.Assert;
28+
import org.junit.Test;
29+
30+
@Slf4j
31+
public class Java9IntHashTest {
32+
33+
private ByteBuf[] generateByteBuffers() {
34+
Random random = new Random();
35+
int hugeDataLen = 4096 * 3;
36+
byte[] hugeData = new byte[hugeDataLen];
37+
for (int i = 0; i < hugeDataLen; i ++) {
38+
hugeData[i] = (byte) (random.nextInt() % 127);
39+
}
40+
41+
// b_total = b1 + b2 + b3;
42+
ByteBuf bTotal = ByteBufAllocator.DEFAULT.heapBuffer(6 + hugeDataLen);
43+
bTotal.writeBytes(new byte[]{1,2,3,4,5,6});
44+
bTotal.writeBytes(hugeData);
45+
ByteBuf b1 = ByteBufAllocator.DEFAULT.heapBuffer(3);
46+
b1.writeBytes(new byte[]{1,2,3});
47+
ByteBuf b2 = ByteBufAllocator.DEFAULT.heapBuffer(3);
48+
b2.writeBytes(new byte[]{4,5,6});
49+
ByteBuf b3 = ByteBufAllocator.DEFAULT.heapBuffer(hugeDataLen);
50+
b3.writeBytes(hugeData);
51+
52+
return new ByteBuf[]{bTotal, b1, new CompositeByteBuf(ByteBufAllocator.DEFAULT, false, 2, b2, b3)};
53+
}
54+
55+
@Test
56+
public void calculateCheckSumUsingCompositeByteBuf() {
57+
// byteBuffers[0] = byteBuffers[1] + byteBuffers[2].
58+
// byteBuffers[2] is a composite ByteBuf.
59+
ByteBuf[] byteBuffers = generateByteBuffers();
60+
ByteBuf bTotal = byteBuffers[0];
61+
ByteBuf b1 = byteBuffers[1];
62+
ByteBuf b2 = byteBuffers[2];
63+
64+
// Calculate: case-1.
65+
int checksumRes1 = Crc32cIntChecksum.computeChecksum(bTotal);
66+
67+
// Calculate: case-2.
68+
int b1CheckSum = Crc32cIntChecksum.computeChecksum(b1);
69+
int checksumRes2 = Crc32cIntChecksum.resumeChecksum(b1CheckSum, b2);
70+
71+
// Verify: the results of both ways to calculate the checksum are same.
72+
Assert.assertEquals(checksumRes1, checksumRes2);
73+
74+
// cleanup.
75+
bTotal.release();
76+
b1.release();
77+
b2.release();
78+
}
79+
80+
@Test
81+
public void calculateCheckSumUsingNoArrayNoMemoryAddrByteBuf() {
82+
// byteBuffers[0] = byteBuffers[1] + byteBuffers[2].
83+
// byteBuffers[2] is a composite ByteBuf.
84+
ByteBuf[] byteBuffers = generateByteBuffers();
85+
ByteBuf bTotal = byteBuffers[0];
86+
ByteBuf b1 = byteBuffers[1];
87+
ByteBuf b2 = new NoArrayNoMemoryAddrByteBuff(byteBuffers[2]);
88+
89+
// Calculate: case-1.
90+
int checksumRes1 = Crc32cIntChecksum.computeChecksum(bTotal);
91+
92+
// Calculate: case-2.
93+
int b1CheckSum = Crc32cIntChecksum.computeChecksum(b1);
94+
int checksumRes2 = Crc32cIntChecksum.resumeChecksum(b1CheckSum, b2);
95+
96+
// Verify: the results of both ways to calculate the checksum are same.
97+
Assert.assertEquals(checksumRes1, checksumRes2);
98+
99+
// cleanup.
100+
bTotal.release();
101+
b1.release();
102+
b2.release();
103+
}
104+
105+
public static class NoArrayNoMemoryAddrByteBuff extends DuplicatedByteBuf {
106+
107+
public NoArrayNoMemoryAddrByteBuff(ByteBuf buffer) {
108+
super(buffer);
109+
}
110+
111+
@Override
112+
public boolean hasArray(){
113+
return false;
114+
}
115+
116+
@Override
117+
public boolean hasMemoryAddress(){
118+
return false;
119+
}
120+
}
121+
}

0 commit comments

Comments
 (0)