Skip to content
This repository was archived by the owner on Mar 3, 2025. It is now read-only.

LRN #601

Open
canacechan wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

LRN #601

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0a8d00a
Feat: GlobalAveragePool
canacechan Mar 12, 2024
0415732
Feat: LRN
canacechan Mar 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/framework/operators/neural-network/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,5 @@ Orion supports currently these `NN` types.
| [`nn.col2im`](nn.col2im.md) | Rearranges column blocks back into a multidimensional image |
| [`nn.conv_transpose`](nn.conv\_transpose.md) | Performs the convolution transpose of the input data tensor and weight tensor. |
| [`nn.conv`](nn.conv.md) | Performs the convolution of the input data tensor and weight tensor. |
| [`nn.lrn`](nn.lrn.md) | Local Response Normalization proposed in the AlexNet paper. It normalizes over local input regions. The local region is defined across the channels. |

75 changes: 75 additions & 0 deletions docs/framework/operators/neural-network/nn.global_average_pool.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# NNTrait::global_average_pool

```rust
fn global_average_pool(tensor: @Tensor<T>) -> Tensor<T>;
```

GlobalAveragePool consumes an input tensor X and applies average pooling across the values in the same channel.
This is equivalent to AveragePool with kernel size equal to the spatial dimension of input tensor.

## Args

* `tensor`(`@Tensor<T>`) - Input data tensor from the previous operator; dimensions for image case are (N x C x H x W), where N is the batch size, C is the number of channels, and H and W are the height and the width of the data. For non image case, the dimensions are in the form of (N x C x D1 x D2 ... Dn), where N is the batch size.

## Returns

* Output data tensor from pooling across the input tensor. The output tensor has the same rank as the input. The first two dimensions of output shape are the same as the input (N x C), while the other dimensions are all 1.

## Examples

```rust
use orion::operators::tensor::{FP8x23Tensor, FP8x23TensorAdd};
use core::array::{ArrayTrait, SpanTrait};
use orion::operators::tensor::{TensorTrait, Tensor};
use orion::utils::{assert_eq, assert_seq_eq};
use orion::operators::tensor::FP8x23TensorPartialEq;
use orion::numbers::{FixedTrait, FP8x23};
use orion::operators::nn::NNTrait;
use orion::operators::nn::FP8x23NN;

fn example() -> Tensor<FP8x23> {
let mut shape = ArrayTrait::<usize>::new();
shape.append(2);
shape.append(4);
shape.append(2);
shape.append(2);

let mut data = ArrayTrait::new();
data.append(FP8x23 { mag: 85392644, sign: false });
data.append(FP8x23 { mag: 61594092, sign: false });
data.append(FP8x23 { mag: 163676643, sign: true });
data.append(FP8x23 { mag: 180530738, sign: false });
data.append(FP8x23 { mag: 168048412, sign: true });
data.append(FP8x23 { mag: 5915510, sign: false });
data.append(FP8x23 { mag: 9047009, sign: true });
data.append(FP8x23 { mag: 46030420, sign: false });
data.append(FP8x23 { mag: 184797857, sign: false });
data.append(FP8x23 { mag: 129370611, sign: false });
data.append(FP8x23 { mag: 174006060, sign: true });
data.append(FP8x23 { mag: 162252480, sign: false });
data.append(FP8x23 { mag: 139240444, sign: true });
data.append(FP8x23 { mag: 168836878, sign: true });
data.append(FP8x23 { mag: 246913333, sign: true });
data.append(FP8x23 { mag: 1047194, sign: true });
data.append(FP8x23 { mag: 238599466, sign: true });
data.append(FP8x23 { mag: 216763643, sign: true });
data.append(FP8x23 { mag: 40581779, sign: true });
data.append(FP8x23 { mag: 209811161, sign: true });
data.append(FP8x23 { mag: 250078311, sign: false });
data.append(FP8x23 { mag: 31811183, sign: true });
data.append(FP8x23 { mag: 36411415, sign: true });
data.append(FP8x23 { mag: 107986324, sign: false });
data.append(FP8x23 { mag: 69727339, sign: false });
data.append(FP8x23 { mag: 223159880, sign: true });
data.append(FP8x23 { mag: 184932087, sign: true });
data.append(FP8x23 { mag: 118617436, sign: false });
data.append(FP8x23 { mag: 134825391, sign: true });
data.append(FP8x23 { mag: 217861279, sign: false });
data.append(FP8x23 { mag: 199069387, sign: false });
data.append(FP8x23 { mag: 192925915, sign: true });
let tensor1 = TensorTrait::new(shape.span(), data.span());

return NNTrait::global_average_pool(@tensor1);
}
>>> [{ mag: 40960207, sign: true } { mag: 31287372, sign: false } { mag: 75603722, sign: true } { mag: 139009462, sign: false } { mag: 176439012, sign: false } { mag: 72460509, sign: true } { mag: 54936798, sign: false } { mag: 22294840, sign: true } ]
```
84 changes: 84 additions & 0 deletions docs/framework/operators/neural-network/nn.lrn.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
# NNTrait::lrn

```rust
fn lrn(tensor: @Tensor<T>, size: usize, alpha: Option<T>, beta: Option<T>, bias: Option<T>) -> Tensor<T>;
```

Local Response Normalization proposed in the AlexNet paper. It normalizes over local input regions. The local region is defined across the channels.

## Args

* `tensor`(`@Tensor<T>`) - Input data tensor from the previous operator; dimensions for image case are (N x C x H x W), where N is the batch size, C is the number of channels, and H and W are the height and the width of the data.
* `size`(usize) - The number of channels to sum over.
* `alpha`(Option<T>) - Scaling parameter.
* `beta`(Option<T>) - The exponent.
* `bias`(Option<T>)

## Returns

* A `Tensor<T>` with the same shape as the input tensor.

## Examples

```rust
use orion::operators::tensor::{FP16x16Tensor, FP16x16TensorAdd};
use core::array::{ArrayTrait, SpanTrait};
use orion::operators::tensor::{TensorTrait, Tensor};
use orion::utils::{assert_eq, assert_seq_eq};
use orion::operators::tensor::FP16x16TensorPartialEq;
use orion::numbers::{FixedTrait, FP16x16};
use orion::operators::nn::NNTrait;
use orion::operators::nn::FP16x16NN;

fn example() -> Tensor<FP16x16> {
let mut shape = ArrayTrait::<usize>::new();
shape.append(3);
shape.append(2);
shape.append(2);
shape.append(3);

let mut data = ArrayTrait::new();
data.append(FP16x16 { mag: 1400832, sign: true });
data.append(FP16x16 { mag: 85184, sign: true });
data.append(FP16x16 { mag: 399616, sign: false });
data.append(FP16x16 { mag: 1828864, sign: false });
data.append(FP16x16 { mag: 962560, sign: true });
data.append(FP16x16 { mag: 1190912, sign: false });
data.append(FP16x16 { mag: 1355776, sign: true });
data.append(FP16x16 { mag: 946176, sign: false });
data.append(FP16x16 { mag: 502528, sign: true });
data.append(FP16x16 { mag: 85824, sign: false });
data.append(FP16x16 { mag: 1333248, sign: false });
data.append(FP16x16 { mag: 39200, sign: true });
data.append(FP16x16 { mag: 712192, sign: false });
data.append(FP16x16 { mag: 1497088, sign: false });
data.append(FP16x16 { mag: 1521664, sign: false });
data.append(FP16x16 { mag: 606720, sign: true });
data.append(FP16x16 { mag: 848384, sign: false });
data.append(FP16x16 { mag: 1732608, sign: true });
data.append(FP16x16 { mag: 1158144, sign: true });
data.append(FP16x16 { mag: 1806336, sign: false });
data.append(FP16x16 { mag: 935424, sign: true });
data.append(FP16x16 { mag: 1106944, sign: true });
data.append(FP16x16 { mag: 1180672, sign: true });
data.append(FP16x16 { mag: 1509376, sign: true });
data.append(FP16x16 { mag: 856064, sign: false });
data.append(FP16x16 { mag: 1841152, sign: false });
data.append(FP16x16 { mag: 75008, sign: false });
data.append(FP16x16 { mag: 19504, sign: true });
data.append(FP16x16 { mag: 1326080, sign: false });
data.append(FP16x16 { mag: 1423360, sign: true });
data.append(FP16x16 { mag: 1258496, sign: true });
data.append(FP16x16 { mag: 671232, sign: true });
data.append(FP16x16 { mag: 548352, sign: false });
data.append(FP16x16 { mag: 797696, sign: true });
data.append(FP16x16 { mag: 1245184, sign: true });
data.append(FP16x16 { mag: 1404928, sign: false });
let tensor1 = TensorTrait::new(shape.span(), data.span());

return NNTrait::lrn(@tensor1, 3, Option::Some(FP16x16 { mag: 13, sign: false }),Option::Some(FP16x16 { mag: 32768, sign: false }),Option::Some(FP16x16 { mag: 131072, sign: false }));
}
>>> [FP16x16 { mag: 964915, sign: true }, FP16x16 { mag: 60080, sign: true }, FP16x16 { mag: 282570, sign: false }, FP16x16 { mag: 1276847, sign: false }, FP16x16 { mag: 680331, sign: true }, FP16x16 { mag: 838194, sign: false }, FP16x16 { mag: 933879, sign: true }, FP16x16 { mag: 667340, sign: false }, FP16x16 { mag: 355340, sign: true }, FP16x16 { mag: 59919, sign: false }, FP16x16 { mag: 942330, sign: false }, FP16x16 { mag: 27589, sign: true }, FP16x16 { mag: 503258, sign: false },
FP16x16 { mag: 1019961, sign: false }, FP16x16 { mag: 1074737, sign: false }, FP16x16 { mag: 424644, sign: true }, FP16x16 { mag: 599690, sign: false }, FP16x16 { mag: 1182030, sign: true }, FP16x16 { mag: 818383, sign: true }, FP16x16 { mag: 1230651, sign: false }, FP16x16 { mag: 660681, sign: true }, FP16x16 { mag: 774752, sign: true }, FP16x16 { mag: 834572, sign: true }, FP16x16 { mag: 1029735, sign: true }, FP16x16 { mag: 605008, sign: false }, FP16x16 { mag: 1295696, sign: false },
FP16x16 { mag: 52967, sign: false }, FP16x16 { mag: 13759, sign: true }, FP16x16 { mag: 937699, sign: false }, FP16x16 { mag: 1006488, sign: true }, FP16x16 { mag: 889419, sign: true }, FP16x16 { mag: 472374, sign: true }, FP16x16 { mag: 387220, sign: false }, FP16x16 { mag: 562768, sign: true }, FP16x16 { mag: 880496, sign: true }, FP16x16 { mag: 993454, sign: false }]
```
110 changes: 110 additions & 0 deletions nodegen/node/global_average_pool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
import numpy as np
from nodegen.node import RunAll
from ..helpers import make_test, to_fp, Tensor, Dtype, FixedImpl, Trait

def global_average_pool(x: np.ndarray) -> np.ndarray:
axis = tuple(range(2, np.ndim(x)))
y = np.average(x, axis=axis)
for _ in axis:
y = np.expand_dims(y, -1)
return y # type: ignore

class Global_average_pool(RunAll):

@staticmethod
def fp8x23_2D():
x = np.random.uniform(-30, 30, (2, 4)).astype(np.float64)
y = global_average_pool(x)

x = Tensor(Dtype.FP8x23, x.shape, to_fp(
x.flatten(), FixedImpl.FP8x23))
y = Tensor(Dtype.FP8x23, y.shape, to_fp(
y.flatten(), FixedImpl.FP8x23))

name = "global_average_pool_fp8x23_2D"
make_test([x], y, "NNTrait::global_average_pool(@input_0)",
name, Trait.NN)

@staticmethod
def fp16x16_2D():
x = np.random.uniform(-30, 30, (3, 2)).astype(np.float16)
y = global_average_pool(x)

x = Tensor(Dtype.FP16x16, x.shape, to_fp(
x.flatten(), FixedImpl.FP16x16))
y = Tensor(Dtype.FP16x16, y.shape, to_fp(
y.flatten(), FixedImpl.FP16x16))

name = "global_average_pool_fp16x16_2D"
make_test([x], y, "NNTrait::global_average_pool(@input_0)",
name, Trait.NN)

@staticmethod
def fp8x23_3D():
x = np.random.uniform(-30, 30, (2, 4, 2)).astype(np.float64)
y = global_average_pool(x)

x = Tensor(Dtype.FP8x23, x.shape, to_fp(
x.flatten(), FixedImpl.FP8x23))
y = Tensor(Dtype.FP8x23, y.shape, to_fp(
y.flatten(), FixedImpl.FP8x23))

name = "global_average_pool_fp8x23_3D"
make_test([x], y, "NNTrait::global_average_pool(@input_0)",
name, Trait.NN)

@staticmethod
def fp16x16_3D():
x = np.random.uniform(-30, 30, (3, 2, 2)).astype(np.float16)
y = global_average_pool(x)

x = Tensor(Dtype.FP16x16, x.shape, to_fp(
x.flatten(), FixedImpl.FP16x16))
y = Tensor(Dtype.FP16x16, y.shape, to_fp(
y.flatten(), FixedImpl.FP16x16))

name = "global_average_pool_fp16x16_3D"
make_test([x], y, "NNTrait::global_average_pool(@input_0)",
name, Trait.NN)

@staticmethod
def fp8x23_4D():
x = np.random.uniform(-30, 30, (2, 4, 2, 2)).astype(np.float64)
y = global_average_pool(x)

x = Tensor(Dtype.FP8x23, x.shape, to_fp(
x.flatten(), FixedImpl.FP8x23))
y = Tensor(Dtype.FP8x23, y.shape, to_fp(
y.flatten(), FixedImpl.FP8x23))

name = "global_average_pool_fp8x23_4D"
make_test([x], y, "NNTrait::global_average_pool(@input_0)",
name, Trait.NN)

@staticmethod
def fp16x16_4D():
x = np.random.uniform(-30, 30, (3, 2, 2, 3)).astype(np.float16)
y = global_average_pool(x)

x = Tensor(Dtype.FP16x16, x.shape, to_fp(
x.flatten(), FixedImpl.FP16x16))
y = Tensor(Dtype.FP16x16, y.shape, to_fp(
y.flatten(), FixedImpl.FP16x16))

name = "global_average_pool_fp16x16_4D"
make_test([x], y, "NNTrait::global_average_pool(@input_0)",
name, Trait.NN)

# @staticmethod
# def fp32x32():
# x = np.random.uniform(-30, 30, (2, 4)).astype(np.float64)
# y = global_average_pool(x)

# x = Tensor(Dtype.FP32x32, x.shape, to_fp(
# x.flatten(), FixedImpl.FP32x32))
# y = Tensor(Dtype.FP32x32, y.shape, to_fp(
# y.flatten(), FixedImpl.FP32x32))

# name = "global_average_pool_fp32x32"
# make_test([x], y, "NNTrait::global_average_pool(@input_0)",
# name, Trait.NN)
69 changes: 69 additions & 0 deletions nodegen/node/lrn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import numpy as np
from nodegen.node import RunAll
import math
from ..helpers import make_test, to_fp, Tensor, Dtype, FixedImpl, Trait

def lrn(x, size, alpha=0.0001, beta=0.75, bias=1.0): # type: ignore
if len(x.shape) != 4:
raise RuntimeError(
f"LRN only applies on 4D tensors but shape is {x.shape!r}."
)
square_sum = np.zeros(x.shape).astype(x.dtype)
minc = x.shape[1]
c1 = int(math.floor((size - 1) / 2))
c2 = int(math.ceil((size - 1) / 2)) + 1
for c in range(x.shape[1]):
begin = max(0, c - c1)
end = min(minc, c + c2)
square_sum[:, c, :, :] = np.sum(x[:, begin:end, :, :] ** 2, axis=1)
y = x / ((bias + (alpha / size) * square_sum) ** beta)
return y.astype(x.dtype)

def get_data_statement(data: np.ndarray, dtype: Dtype) -> list[str]:
match dtype:
case Dtype.FP8x23:
return ["Option::Some(FP8x23 { "+f"mag: {abs(int(x))}, sign: {str(x < 0).lower()} "+"})" for x in data.flatten()]
case Dtype.FP16x16:
return ["Option::Some(FP16x16 { "+f"mag: {abs(int(x))}, sign: {str(x < 0).lower()} "+"})" for x in data.flatten()]

class Lrn(RunAll):
@staticmethod
def fp8x23():
alpha = 0.0002
beta = 0.5
bias = 2.0
args = [alpha, beta, bias]
args_str = get_data_statement(to_fp(np.array(args).flatten(), FixedImpl.FP8x23), Dtype.FP8x23)
nsize = 3
x = np.random.uniform(-30, 30, (2, 4, 2, 2)).astype(np.float64)
y = lrn(x, nsize, *args)

x = Tensor(Dtype.FP8x23, x.shape, to_fp(
x.flatten(), FixedImpl.FP8x23))
y = Tensor(Dtype.FP8x23, y.shape, to_fp(
y.flatten(), FixedImpl.FP8x23))

name = "lrn_fp8x23"
make_test([x], y, f"NNTrait::lrn(@input_0, {nsize}, {','.join(args_str)})",
name, Trait.NN)

@staticmethod
def fp16x16():
alpha = 0.0002
beta = 0.5
bias = 2.0
args = [alpha, beta, bias]
args_str = get_data_statement(to_fp(np.array(args).flatten(), FixedImpl.FP16x16), Dtype.FP16x16)
nsize = 3
x = np.random.uniform(-30, 30, (3, 2, 2, 3)).astype(np.float16)
y = lrn(x, nsize, *args)

x = Tensor(Dtype.FP16x16, x.shape, to_fp(
x.flatten(), FixedImpl.FP16x16))
y = Tensor(Dtype.FP16x16, y.shape, to_fp(
y.flatten(), FixedImpl.FP16x16))

name = "lrn_fp16x16"
make_test([x], y, f"NNTrait::lrn(@input_0, {nsize}, {','.join(args_str)})",
name, Trait.NN)

Loading