From 0a8d00a0edbd4d45ded76ee7e6a4f4d8654f6990 Mon Sep 17 00:00:00 2001 From: Canance Chan Date: Tue, 12 Mar 2024 11:12:42 +0000 Subject: [PATCH 1/2] Feat: GlobalAveragePool --- .../operators/neural-network/README.md | 1 + .../neural-network/nn.global_average_pool.md | 75 ++++++++++++ nodegen/node/global_average_pool.py | 110 ++++++++++++++++++ src/operators/nn/core.cairo | 78 +++++++++++++ src/operators/nn/functional.cairo | 1 + .../nn/functional/global_average_pool.cairo | 63 ++++++++++ .../nn/implementations/nn_fp16x16.cairo | 4 + .../nn/implementations/nn_fp32x32.cairo | 4 + .../nn/implementations/nn_fp64x64.cairo | 4 + .../nn/implementations/nn_fp8x23.cairo | 4 + src/operators/nn/implementations/nn_i32.cairo | 4 + src/operators/nn/implementations/nn_i8.cairo | 4 + src/operators/nn/implementations/nn_u32.cairo | 4 + tests/nodes.cairo | 7 ++ .../global_average_pool_fp16x16_2D.cairo | 20 ++++ .../input_0.cairo | 19 +++ .../output_0.cairo | 19 +++ .../global_average_pool_fp16x16_3D.cairo | 20 ++++ .../input_0.cairo | 26 +++++ .../output_0.cairo | 20 ++++ .../global_average_pool_fp16x16_4D.cairo | 20 ++++ .../input_0.cairo | 51 ++++++++ .../output_0.cairo | 21 ++++ .../nodes/global_average_pool_fp8x23_2D.cairo | 20 ++++ .../input_0.cairo | 21 ++++ .../output_0.cairo | 21 ++++ .../nodes/global_average_pool_fp8x23_3D.cairo | 20 ++++ .../input_0.cairo | 30 +++++ .../output_0.cairo | 22 ++++ .../nodes/global_average_pool_fp8x23_4D.cairo | 20 ++++ .../input_0.cairo | 47 ++++++++ .../output_0.cairo | 23 ++++ 32 files changed, 803 insertions(+) create mode 100644 docs/framework/operators/neural-network/nn.global_average_pool.md create mode 100644 nodegen/node/global_average_pool.py create mode 100644 src/operators/nn/functional/global_average_pool.cairo create mode 100644 tests/nodes/global_average_pool_fp16x16_2D.cairo create mode 100644 tests/nodes/global_average_pool_fp16x16_2D/input_0.cairo create mode 100644 tests/nodes/global_average_pool_fp16x16_2D/output_0.cairo create mode 100644 tests/nodes/global_average_pool_fp16x16_3D.cairo create mode 100644 tests/nodes/global_average_pool_fp16x16_3D/input_0.cairo create mode 100644 tests/nodes/global_average_pool_fp16x16_3D/output_0.cairo create mode 100644 tests/nodes/global_average_pool_fp16x16_4D.cairo create mode 100644 tests/nodes/global_average_pool_fp16x16_4D/input_0.cairo create mode 100644 tests/nodes/global_average_pool_fp16x16_4D/output_0.cairo create mode 100644 tests/nodes/global_average_pool_fp8x23_2D.cairo create mode 100644 tests/nodes/global_average_pool_fp8x23_2D/input_0.cairo create mode 100644 tests/nodes/global_average_pool_fp8x23_2D/output_0.cairo create mode 100644 tests/nodes/global_average_pool_fp8x23_3D.cairo create mode 100644 tests/nodes/global_average_pool_fp8x23_3D/input_0.cairo create mode 100644 tests/nodes/global_average_pool_fp8x23_3D/output_0.cairo create mode 100644 tests/nodes/global_average_pool_fp8x23_4D.cairo create mode 100644 tests/nodes/global_average_pool_fp8x23_4D/input_0.cairo create mode 100644 tests/nodes/global_average_pool_fp8x23_4D/output_0.cairo diff --git a/docs/framework/operators/neural-network/README.md b/docs/framework/operators/neural-network/README.md index fc3bfb612..1dd628070 100644 --- a/docs/framework/operators/neural-network/README.md +++ b/docs/framework/operators/neural-network/README.md @@ -39,4 +39,5 @@ Orion supports currently these `NN` types. | [`nn.col2im`](nn.col2im.md) | Rearranges column blocks back into a multidimensional image | | [`nn.conv_transpose`](nn.conv\_transpose.md) | Performs the convolution transpose of the input data tensor and weight tensor. | | [`nn.conv`](nn.conv.md) | Performs the convolution of the input data tensor and weight tensor. | +| [`nn.global_average_pool`](nn.global\_average\_pool.md) | GlobalAveragePool consumes an input tensor X and applies average pooling across the values in the same channel. | diff --git a/docs/framework/operators/neural-network/nn.global_average_pool.md b/docs/framework/operators/neural-network/nn.global_average_pool.md new file mode 100644 index 000000000..b9a29b28e --- /dev/null +++ b/docs/framework/operators/neural-network/nn.global_average_pool.md @@ -0,0 +1,75 @@ +# NNTrait::global_average_pool + +```rust + fn global_average_pool(tensor: @Tensor) -> Tensor; +``` + +GlobalAveragePool consumes an input tensor X and applies average pooling across the values in the same channel. +This is equivalent to AveragePool with kernel size equal to the spatial dimension of input tensor. + +## Args + +* `tensor`(`@Tensor`) - Input data tensor from the previous operator; dimensions for image case are (N x C x H x W), where N is the batch size, C is the number of channels, and H and W are the height and the width of the data. For non image case, the dimensions are in the form of (N x C x D1 x D2 ... Dn), where N is the batch size. + +## Returns + +* Output data tensor from pooling across the input tensor. The output tensor has the same rank as the input. The first two dimensions of output shape are the same as the input (N x C), while the other dimensions are all 1. + +## Examples + +```rust +use orion::operators::tensor::{FP8x23Tensor, FP8x23TensorAdd}; +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::utils::{assert_eq, assert_seq_eq}; +use orion::operators::tensor::FP8x23TensorPartialEq; +use orion::numbers::{FixedTrait, FP8x23}; +use orion::operators::nn::NNTrait; +use orion::operators::nn::FP8x23NN; + +fn example() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(2); + shape.append(4); + shape.append(2); + shape.append(2); + + let mut data = ArrayTrait::new(); + data.append(FP8x23 { mag: 85392644, sign: false }); + data.append(FP8x23 { mag: 61594092, sign: false }); + data.append(FP8x23 { mag: 163676643, sign: true }); + data.append(FP8x23 { mag: 180530738, sign: false }); + data.append(FP8x23 { mag: 168048412, sign: true }); + data.append(FP8x23 { mag: 5915510, sign: false }); + data.append(FP8x23 { mag: 9047009, sign: true }); + data.append(FP8x23 { mag: 46030420, sign: false }); + data.append(FP8x23 { mag: 184797857, sign: false }); + data.append(FP8x23 { mag: 129370611, sign: false }); + data.append(FP8x23 { mag: 174006060, sign: true }); + data.append(FP8x23 { mag: 162252480, sign: false }); + data.append(FP8x23 { mag: 139240444, sign: true }); + data.append(FP8x23 { mag: 168836878, sign: true }); + data.append(FP8x23 { mag: 246913333, sign: true }); + data.append(FP8x23 { mag: 1047194, sign: true }); + data.append(FP8x23 { mag: 238599466, sign: true }); + data.append(FP8x23 { mag: 216763643, sign: true }); + data.append(FP8x23 { mag: 40581779, sign: true }); + data.append(FP8x23 { mag: 209811161, sign: true }); + data.append(FP8x23 { mag: 250078311, sign: false }); + data.append(FP8x23 { mag: 31811183, sign: true }); + data.append(FP8x23 { mag: 36411415, sign: true }); + data.append(FP8x23 { mag: 107986324, sign: false }); + data.append(FP8x23 { mag: 69727339, sign: false }); + data.append(FP8x23 { mag: 223159880, sign: true }); + data.append(FP8x23 { mag: 184932087, sign: true }); + data.append(FP8x23 { mag: 118617436, sign: false }); + data.append(FP8x23 { mag: 134825391, sign: true }); + data.append(FP8x23 { mag: 217861279, sign: false }); + data.append(FP8x23 { mag: 199069387, sign: false }); + data.append(FP8x23 { mag: 192925915, sign: true }); + let tensor1 = TensorTrait::new(shape.span(), data.span()); + + return NNTrait::global_average_pool(@tensor1); +} +>>> [{ mag: 40960207, sign: true } { mag: 31287372, sign: false } { mag: 75603722, sign: true } { mag: 139009462, sign: false } { mag: 176439012, sign: false } { mag: 72460509, sign: true } { mag: 54936798, sign: false } { mag: 22294840, sign: true } ] +``` diff --git a/nodegen/node/global_average_pool.py b/nodegen/node/global_average_pool.py new file mode 100644 index 000000000..46718aca2 --- /dev/null +++ b/nodegen/node/global_average_pool.py @@ -0,0 +1,110 @@ +import numpy as np +from nodegen.node import RunAll +from ..helpers import make_test, to_fp, Tensor, Dtype, FixedImpl, Trait + +def global_average_pool(x: np.ndarray) -> np.ndarray: + axis = tuple(range(2, np.ndim(x))) + y = np.average(x, axis=axis) + for _ in axis: + y = np.expand_dims(y, -1) + return y # type: ignore + +class Global_average_pool(RunAll): + + @staticmethod + def fp8x23_2D(): + x = np.random.uniform(-30, 30, (2, 4)).astype(np.float64) + y = global_average_pool(x) + + x = Tensor(Dtype.FP8x23, x.shape, to_fp( + x.flatten(), FixedImpl.FP8x23)) + y = Tensor(Dtype.FP8x23, y.shape, to_fp( + y.flatten(), FixedImpl.FP8x23)) + + name = "global_average_pool_fp8x23_2D" + make_test([x], y, "NNTrait::global_average_pool(@input_0)", + name, Trait.NN) + + @staticmethod + def fp16x16_2D(): + x = np.random.uniform(-30, 30, (3, 2)).astype(np.float16) + y = global_average_pool(x) + + x = Tensor(Dtype.FP16x16, x.shape, to_fp( + x.flatten(), FixedImpl.FP16x16)) + y = Tensor(Dtype.FP16x16, y.shape, to_fp( + y.flatten(), FixedImpl.FP16x16)) + + name = "global_average_pool_fp16x16_2D" + make_test([x], y, "NNTrait::global_average_pool(@input_0)", + name, Trait.NN) + + @staticmethod + def fp8x23_3D(): + x = np.random.uniform(-30, 30, (2, 4, 2)).astype(np.float64) + y = global_average_pool(x) + + x = Tensor(Dtype.FP8x23, x.shape, to_fp( + x.flatten(), FixedImpl.FP8x23)) + y = Tensor(Dtype.FP8x23, y.shape, to_fp( + y.flatten(), FixedImpl.FP8x23)) + + name = "global_average_pool_fp8x23_3D" + make_test([x], y, "NNTrait::global_average_pool(@input_0)", + name, Trait.NN) + + @staticmethod + def fp16x16_3D(): + x = np.random.uniform(-30, 30, (3, 2, 2)).astype(np.float16) + y = global_average_pool(x) + + x = Tensor(Dtype.FP16x16, x.shape, to_fp( + x.flatten(), FixedImpl.FP16x16)) + y = Tensor(Dtype.FP16x16, y.shape, to_fp( + y.flatten(), FixedImpl.FP16x16)) + + name = "global_average_pool_fp16x16_3D" + make_test([x], y, "NNTrait::global_average_pool(@input_0)", + name, Trait.NN) + + @staticmethod + def fp8x23_4D(): + x = np.random.uniform(-30, 30, (2, 4, 2, 2)).astype(np.float64) + y = global_average_pool(x) + + x = Tensor(Dtype.FP8x23, x.shape, to_fp( + x.flatten(), FixedImpl.FP8x23)) + y = Tensor(Dtype.FP8x23, y.shape, to_fp( + y.flatten(), FixedImpl.FP8x23)) + + name = "global_average_pool_fp8x23_4D" + make_test([x], y, "NNTrait::global_average_pool(@input_0)", + name, Trait.NN) + + @staticmethod + def fp16x16_4D(): + x = np.random.uniform(-30, 30, (3, 2, 2, 3)).astype(np.float16) + y = global_average_pool(x) + + x = Tensor(Dtype.FP16x16, x.shape, to_fp( + x.flatten(), FixedImpl.FP16x16)) + y = Tensor(Dtype.FP16x16, y.shape, to_fp( + y.flatten(), FixedImpl.FP16x16)) + + name = "global_average_pool_fp16x16_4D" + make_test([x], y, "NNTrait::global_average_pool(@input_0)", + name, Trait.NN) + + # @staticmethod + # def fp32x32(): + # x = np.random.uniform(-30, 30, (2, 4)).astype(np.float64) + # y = global_average_pool(x) + + # x = Tensor(Dtype.FP32x32, x.shape, to_fp( + # x.flatten(), FixedImpl.FP32x32)) + # y = Tensor(Dtype.FP32x32, y.shape, to_fp( + # y.flatten(), FixedImpl.FP32x32)) + + # name = "global_average_pool_fp32x32" + # make_test([x], y, "NNTrait::global_average_pool(@input_0)", + # name, Trait.NN) diff --git a/src/operators/nn/core.cairo b/src/operators/nn/core.cairo index 35d318b28..1b75f7ccb 100644 --- a/src/operators/nn/core.cairo +++ b/src/operators/nn/core.cairo @@ -18,6 +18,7 @@ use orion::operators::tensor::core::Tensor; /// col2im - Rearranges column blocks back into a multidimensional image /// conv_transpose - Performs the convolution transpose of the input data tensor and weight tensor. /// conv - Performs the convolution of the input data tensor and weight tensor. +/// global_average_pool - GlobalAveragePool consumes an input tensor X and applies average pooling across the values in the same channel. trait NNTrait { /// # NNTrait::relu /// @@ -1304,4 +1305,81 @@ trait NNTrait { mode: Option, padding_mode: Option, ) -> Tensor; + /// # NNTrait::global_average_pool + /// + /// ```rust + /// fn global_average_pool(tensor: @Tensor) -> Tensor; + /// ``` + /// + /// GlobalAveragePool consumes an input tensor X and applies average pooling across the values in the same channel. + /// This is equivalent to AveragePool with kernel size equal to the spatial dimension of input tensor. + /// + /// ## Args + /// + /// * `tensor`(`@Tensor`) - Input data tensor from the previous operator; dimensions for image case are (N x C x H x W), where N is the batch size, C is the number of channels, and H and W are the height and the width of the data. For non image case, the dimensions are in the form of (N x C x D1 x D2 ... Dn), where N is the batch size. + /// + /// ## Returns + /// + /// * Output data tensor from pooling across the input tensor. The output tensor has the same rank as the input. The first two dimensions of output shape are the same as the input (N x C), while the other dimensions are all 1. + /// + /// ## Examples + /// + /// ```rust + /// use orion::operators::tensor::{FP8x23Tensor, FP8x23TensorAdd}; + /// use core::array::{ArrayTrait, SpanTrait}; + /// use orion::operators::tensor::{TensorTrait, Tensor}; + /// use orion::utils::{assert_eq, assert_seq_eq}; + /// use orion::operators::tensor::FP8x23TensorPartialEq; + /// use orion::numbers::{FixedTrait, FP8x23}; + /// use orion::operators::nn::NNTrait; + /// use orion::operators::nn::FP8x23NN; + /// + /// fn example() -> Tensor { + /// let mut shape = ArrayTrait::::new(); + /// shape.append(2); + /// shape.append(4); + /// shape.append(2); + /// shape.append(2); + /// + /// let mut data = ArrayTrait::new(); + /// data.append(FP8x23 { mag: 85392644, sign: false }); + /// data.append(FP8x23 { mag: 61594092, sign: false }); + /// data.append(FP8x23 { mag: 163676643, sign: true }); + /// data.append(FP8x23 { mag: 180530738, sign: false }); + /// data.append(FP8x23 { mag: 168048412, sign: true }); + /// data.append(FP8x23 { mag: 5915510, sign: false }); + /// data.append(FP8x23 { mag: 9047009, sign: true }); + /// data.append(FP8x23 { mag: 46030420, sign: false }); + /// data.append(FP8x23 { mag: 184797857, sign: false }); + /// data.append(FP8x23 { mag: 129370611, sign: false }); + /// data.append(FP8x23 { mag: 174006060, sign: true }); + /// data.append(FP8x23 { mag: 162252480, sign: false }); + /// data.append(FP8x23 { mag: 139240444, sign: true }); + /// data.append(FP8x23 { mag: 168836878, sign: true }); + /// data.append(FP8x23 { mag: 246913333, sign: true }); + /// data.append(FP8x23 { mag: 1047194, sign: true }); + /// data.append(FP8x23 { mag: 238599466, sign: true }); + /// data.append(FP8x23 { mag: 216763643, sign: true }); + /// data.append(FP8x23 { mag: 40581779, sign: true }); + /// data.append(FP8x23 { mag: 209811161, sign: true }); + /// data.append(FP8x23 { mag: 250078311, sign: false }); + /// data.append(FP8x23 { mag: 31811183, sign: true }); + /// data.append(FP8x23 { mag: 36411415, sign: true }); + /// data.append(FP8x23 { mag: 107986324, sign: false }); + /// data.append(FP8x23 { mag: 69727339, sign: false }); + /// data.append(FP8x23 { mag: 223159880, sign: true }); + /// data.append(FP8x23 { mag: 184932087, sign: true }); + /// data.append(FP8x23 { mag: 118617436, sign: false }); + /// data.append(FP8x23 { mag: 134825391, sign: true }); + /// data.append(FP8x23 { mag: 217861279, sign: false }); + /// data.append(FP8x23 { mag: 199069387, sign: false }); + /// data.append(FP8x23 { mag: 192925915, sign: true }); + /// let tensor1 = TensorTrait::new(shape.span(), data.span()); + /// + /// return NNTrait::global_average_pool(@tensor1); + /// } + /// >>> [{ mag: 40960207, sign: true } { mag: 31287372, sign: false } { mag: 75603722, sign: true } { mag: 139009462, sign: false } { mag: 176439012, sign: false } { mag: 72460509, sign: true } { mag: 54936798, sign: false } { mag: 22294840, sign: true } ] + /// ``` + /// + fn global_average_pool(tensor: @Tensor) -> Tensor; } diff --git a/src/operators/nn/functional.cairo b/src/operators/nn/functional.cairo index 45e1c1ec9..b1734558e 100644 --- a/src/operators/nn/functional.cairo +++ b/src/operators/nn/functional.cairo @@ -16,3 +16,4 @@ mod conv_transpose; mod depth_to_space; mod space_to_depth; mod conv; +mod global_average_pool; diff --git a/src/operators/nn/functional/global_average_pool.cairo b/src/operators/nn/functional/global_average_pool.cairo new file mode 100644 index 000000000..e7381e370 --- /dev/null +++ b/src/operators/nn/functional/global_average_pool.cairo @@ -0,0 +1,63 @@ +use orion::numbers::fixed_point::core::FixedTrait; +use orion::numbers::NumberTrait; +use orion::operators::tensor::core::{Tensor, TensorTrait}; +use orion::operators::tensor::helpers::{reduce_output_shape, len_from_shape, combine_indices}; +use orion::operators::tensor::math::{reduce_sum::accumulate_sum, arithmetic::div_downcast}; + + +fn global_average_pool< + T, + MAG, + impl TTensor: TensorTrait, + impl TNumber: NumberTrait, + impl TAdd: Add, + impl TSub: Sub, + impl TMul: Mul, + impl TDiv: Div, + impl TTensorAdd: Add>, + impl TPartialOrd: PartialOrd, + impl TPartialEq: PartialEq, + impl TAddEq: AddEq, + impl TCopy: Copy, + impl TDrop: Drop, +>( + tensor: Tensor +) -> Tensor { + assert((tensor.shape).len() >= 2, 'Unexpected shape.'); + let N = *(tensor.shape).at(0); + let C = *(tensor.shape).at(1); + let mut shape = array![N, C]; + let mut i:usize = 2; + let one: T = NumberTrait::one(); + let len = (tensor.shape).len(); + let mut num: usize = 1; + let mut num_t: T = one; + while i < len { + shape.append(1); + let v = *(tensor.shape).at(i); + let mut v_t: T = one; + let mut j: usize = 1; + while j != v { + v_t = v_t + one; + j += 1; + }; + num *= v; + num_t = num_t * v_t; + i += 1; + }; + let mut arr: Array = array![]; + i = 0; + let tensor_len = (tensor.data).len(); + while i < tensor_len { + let mut j:usize = 0; + let mut r: T = NumberTrait::zero(); + while j != num { + r += *(tensor.data).at(i); + j += 1; + i += 1; + }; + arr.append(r / num_t); + }; + + TensorTrait::::new(shape.span(), arr.span()) +} diff --git a/src/operators/nn/implementations/nn_fp16x16.cairo b/src/operators/nn/implementations/nn_fp16x16.cairo index 1c018ade3..758dc2331 100644 --- a/src/operators/nn/implementations/nn_fp16x16.cairo +++ b/src/operators/nn/implementations/nn_fp16x16.cairo @@ -143,4 +143,8 @@ impl FP16x16NN of NNTrait { ) -> Tensor { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } + + fn global_average_pool(tensor: @Tensor) -> Tensor { + functional::global_average_pool::global_average_pool(*tensor) + } } diff --git a/src/operators/nn/implementations/nn_fp32x32.cairo b/src/operators/nn/implementations/nn_fp32x32.cairo index a5725eccb..b8ead0f91 100644 --- a/src/operators/nn/implementations/nn_fp32x32.cairo +++ b/src/operators/nn/implementations/nn_fp32x32.cairo @@ -137,4 +137,8 @@ impl FP32x32NN of NNTrait { ) -> Tensor { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } + + fn global_average_pool(tensor: @Tensor) -> Tensor { + functional::global_average_pool::global_average_pool(*tensor) + } } diff --git a/src/operators/nn/implementations/nn_fp64x64.cairo b/src/operators/nn/implementations/nn_fp64x64.cairo index 01a3b30ad..2d857d7d7 100644 --- a/src/operators/nn/implementations/nn_fp64x64.cairo +++ b/src/operators/nn/implementations/nn_fp64x64.cairo @@ -137,4 +137,8 @@ impl FP64x64NN of NNTrait { ) -> Tensor { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } + + fn global_average_pool(tensor: @Tensor) -> Tensor { + functional::global_average_pool::global_average_pool(*tensor) + } } diff --git a/src/operators/nn/implementations/nn_fp8x23.cairo b/src/operators/nn/implementations/nn_fp8x23.cairo index d80d2c323..e49774954 100644 --- a/src/operators/nn/implementations/nn_fp8x23.cairo +++ b/src/operators/nn/implementations/nn_fp8x23.cairo @@ -139,4 +139,8 @@ impl FP8x23NN of NNTrait { ) -> Tensor { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } + + fn global_average_pool(tensor: @Tensor) -> Tensor { + functional::global_average_pool::global_average_pool(*tensor) + } } diff --git a/src/operators/nn/implementations/nn_i32.cairo b/src/operators/nn/implementations/nn_i32.cairo index 29a94d288..def7aa7d8 100644 --- a/src/operators/nn/implementations/nn_i32.cairo +++ b/src/operators/nn/implementations/nn_i32.cairo @@ -130,4 +130,8 @@ impl I32NN of NNTrait { ) -> Tensor { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } + + fn global_average_pool(tensor: @Tensor) -> Tensor { + panic(array!['not supported!']) + } } diff --git a/src/operators/nn/implementations/nn_i8.cairo b/src/operators/nn/implementations/nn_i8.cairo index e22de6b43..d1771a631 100644 --- a/src/operators/nn/implementations/nn_i8.cairo +++ b/src/operators/nn/implementations/nn_i8.cairo @@ -130,4 +130,8 @@ impl I8NN of NNTrait { ) -> Tensor { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } + + fn global_average_pool(tensor: @Tensor) -> Tensor { + panic(array!['not supported!']) + } } diff --git a/src/operators/nn/implementations/nn_u32.cairo b/src/operators/nn/implementations/nn_u32.cairo index 7352b7ad9..67072c4f0 100644 --- a/src/operators/nn/implementations/nn_u32.cairo +++ b/src/operators/nn/implementations/nn_u32.cairo @@ -130,4 +130,8 @@ impl U32NN of NNTrait { ) -> Tensor { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } + + fn global_average_pool(tensor: @Tensor) -> Tensor { + panic(array!['not supported!']) + } } diff --git a/tests/nodes.cairo b/tests/nodes.cairo index 29bebb762..bc8a9ea7d 100644 --- a/tests/nodes.cairo +++ b/tests/nodes.cairo @@ -1047,3 +1047,10 @@ mod label_encoder_fp8x23_default; mod label_encoder_i8_default; mod label_encoder_i32_default; mod label_encoder_u32_default; +mod global_average_pool_fp16x16_2D; +mod global_average_pool_fp16x16_3D; +mod global_average_pool_fp16x16_4D; +mod global_average_pool_fp8x23_2D; +mod global_average_pool_fp8x23_3D; +mod global_average_pool_fp8x23_4D; + diff --git a/tests/nodes/global_average_pool_fp16x16_2D.cairo b/tests/nodes/global_average_pool_fp16x16_2D.cairo new file mode 100644 index 000000000..c6e244c65 --- /dev/null +++ b/tests/nodes/global_average_pool_fp16x16_2D.cairo @@ -0,0 +1,20 @@ +mod input_0; +mod output_0; + + +use orion::operators::nn::FP16x16NN; +use orion::operators::nn::NNTrait; +use orion::numbers::FixedTrait; +use orion::operators::tensor::FP16x16TensorPartialEq; +use orion::utils::{assert_eq, assert_seq_eq}; + +#[test] +#[available_gas(2000000000)] +fn test_global_average_pool_fp16x16_2D() { + let input_0 = input_0::input_0(); + let z_0 = output_0::output_0(); + + let y_0 = NNTrait::global_average_pool(@input_0); + + assert_eq(y_0, z_0); +} diff --git a/tests/nodes/global_average_pool_fp16x16_2D/input_0.cairo b/tests/nodes/global_average_pool_fp16x16_2D/input_0.cairo new file mode 100644 index 000000000..6d46e72d3 --- /dev/null +++ b/tests/nodes/global_average_pool_fp16x16_2D/input_0.cairo @@ -0,0 +1,19 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP16x16Tensor, FP16x16TensorAdd}; +use orion::numbers::{FixedTrait, FP16x16}; + +fn input_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(3); + shape.append(2); + + let mut data = ArrayTrait::new(); + data.append(FP16x16 { mag: 1394688, sign: false }); + data.append(FP16x16 { mag: 292608, sign: true }); + data.append(FP16x16 { mag: 427008, sign: false }); + data.append(FP16x16 { mag: 11840, sign: false }); + data.append(FP16x16 { mag: 736256, sign: true }); + data.append(FP16x16 { mag: 18736, sign: true }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/global_average_pool_fp16x16_2D/output_0.cairo b/tests/nodes/global_average_pool_fp16x16_2D/output_0.cairo new file mode 100644 index 000000000..ecc8bf593 --- /dev/null +++ b/tests/nodes/global_average_pool_fp16x16_2D/output_0.cairo @@ -0,0 +1,19 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP16x16Tensor, FP16x16TensorAdd}; +use orion::numbers::{FixedTrait, FP16x16}; + +fn output_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(3); + shape.append(2); + + let mut data = ArrayTrait::new(); + data.append(FP16x16 { mag: 1394688, sign: false }); + data.append(FP16x16 { mag: 292608, sign: true }); + data.append(FP16x16 { mag: 427008, sign: false }); + data.append(FP16x16 { mag: 11840, sign: false }); + data.append(FP16x16 { mag: 736256, sign: true }); + data.append(FP16x16 { mag: 18736, sign: true }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/global_average_pool_fp16x16_3D.cairo b/tests/nodes/global_average_pool_fp16x16_3D.cairo new file mode 100644 index 000000000..e65a4f6c4 --- /dev/null +++ b/tests/nodes/global_average_pool_fp16x16_3D.cairo @@ -0,0 +1,20 @@ +mod input_0; +mod output_0; + + +use orion::operators::nn::FP16x16NN; +use orion::operators::nn::NNTrait; +use orion::numbers::FixedTrait; +use orion::operators::tensor::FP16x16TensorPartialEq; +use orion::utils::{assert_eq, assert_seq_eq}; + +#[test] +#[available_gas(2000000000)] +fn test_global_average_pool_fp16x16_3D() { + let input_0 = input_0::input_0(); + let z_0 = output_0::output_0(); + + let y_0 = NNTrait::global_average_pool(@input_0); + + assert_eq(y_0, z_0); +} diff --git a/tests/nodes/global_average_pool_fp16x16_3D/input_0.cairo b/tests/nodes/global_average_pool_fp16x16_3D/input_0.cairo new file mode 100644 index 000000000..c52cd4fcd --- /dev/null +++ b/tests/nodes/global_average_pool_fp16x16_3D/input_0.cairo @@ -0,0 +1,26 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP16x16Tensor, FP16x16TensorAdd}; +use orion::numbers::{FixedTrait, FP16x16}; + +fn input_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(3); + shape.append(2); + shape.append(2); + + let mut data = ArrayTrait::new(); + data.append(FP16x16 { mag: 1927168, sign: false }); + data.append(FP16x16 { mag: 736768, sign: false }); + data.append(FP16x16 { mag: 1823744, sign: false }); + data.append(FP16x16 { mag: 831488, sign: true }); + data.append(FP16x16 { mag: 768512, sign: false }); + data.append(FP16x16 { mag: 1181696, sign: true }); + data.append(FP16x16 { mag: 1544192, sign: true }); + data.append(FP16x16 { mag: 1265664, sign: false }); + data.append(FP16x16 { mag: 1175552, sign: true }); + data.append(FP16x16 { mag: 261376, sign: true }); + data.append(FP16x16 { mag: 1157120, sign: false }); + data.append(FP16x16 { mag: 812032, sign: false }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/global_average_pool_fp16x16_3D/output_0.cairo b/tests/nodes/global_average_pool_fp16x16_3D/output_0.cairo new file mode 100644 index 000000000..51dc1abef --- /dev/null +++ b/tests/nodes/global_average_pool_fp16x16_3D/output_0.cairo @@ -0,0 +1,20 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP16x16Tensor, FP16x16TensorAdd}; +use orion::numbers::{FixedTrait, FP16x16}; + +fn output_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(3); + shape.append(2); + shape.append(1); + + let mut data = ArrayTrait::new(); + data.append(FP16x16 { mag: 1332224, sign: false }); + data.append(FP16x16 { mag: 496128, sign: false }); + data.append(FP16x16 { mag: 206592, sign: true }); + data.append(FP16x16 { mag: 139264, sign: true }); + data.append(FP16x16 { mag: 718336, sign: true }); + data.append(FP16x16 { mag: 984576, sign: false }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/global_average_pool_fp16x16_4D.cairo b/tests/nodes/global_average_pool_fp16x16_4D.cairo new file mode 100644 index 000000000..41917e388 --- /dev/null +++ b/tests/nodes/global_average_pool_fp16x16_4D.cairo @@ -0,0 +1,20 @@ +mod input_0; +mod output_0; + + +use orion::operators::nn::FP16x16NN; +use orion::operators::nn::NNTrait; +use orion::numbers::FixedTrait; +use orion::operators::tensor::FP16x16TensorPartialEq; +use orion::utils::{assert_eq, assert_seq_eq}; + +#[test] +#[available_gas(2000000000)] +fn test_global_average_pool_fp16x16_4D() { + let input_0 = input_0::input_0(); + let z_0 = output_0::output_0(); + + let y_0 = NNTrait::global_average_pool(@input_0); + + assert_eq(y_0, z_0); +} diff --git a/tests/nodes/global_average_pool_fp16x16_4D/input_0.cairo b/tests/nodes/global_average_pool_fp16x16_4D/input_0.cairo new file mode 100644 index 000000000..9a3b6c688 --- /dev/null +++ b/tests/nodes/global_average_pool_fp16x16_4D/input_0.cairo @@ -0,0 +1,51 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP16x16Tensor, FP16x16TensorAdd}; +use orion::numbers::{FixedTrait, FP16x16}; + +fn input_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(3); + shape.append(2); + shape.append(2); + shape.append(3); + + let mut data = ArrayTrait::new(); + data.append(FP16x16 { mag: 1092608, sign: false }); + data.append(FP16x16 { mag: 634368, sign: false }); + data.append(FP16x16 { mag: 1613824, sign: true }); + data.append(FP16x16 { mag: 947200, sign: false }); + data.append(FP16x16 { mag: 819200, sign: true }); + data.append(FP16x16 { mag: 1251328, sign: true }); + data.append(FP16x16 { mag: 1854464, sign: false }); + data.append(FP16x16 { mag: 1760256, sign: true }); + data.append(FP16x16 { mag: 135168, sign: false }); + data.append(FP16x16 { mag: 88064, sign: false }); + data.append(FP16x16 { mag: 924672, sign: true }); + data.append(FP16x16 { mag: 1415168, sign: false }); + data.append(FP16x16 { mag: 862208, sign: false }); + data.append(FP16x16 { mag: 673280, sign: false }); + data.append(FP16x16 { mag: 857600, sign: true }); + data.append(FP16x16 { mag: 1040384, sign: false }); + data.append(FP16x16 { mag: 1608704, sign: false }); + data.append(FP16x16 { mag: 1477632, sign: false }); + data.append(FP16x16 { mag: 1648640, sign: true }); + data.append(FP16x16 { mag: 1811456, sign: true }); + data.append(FP16x16 { mag: 582656, sign: true }); + data.append(FP16x16 { mag: 1102848, sign: false }); + data.append(FP16x16 { mag: 734208, sign: true }); + data.append(FP16x16 { mag: 1461248, sign: false }); + data.append(FP16x16 { mag: 151680, sign: true }); + data.append(FP16x16 { mag: 1010688, sign: true }); + data.append(FP16x16 { mag: 1299456, sign: true }); + data.append(FP16x16 { mag: 681472, sign: false }); + data.append(FP16x16 { mag: 418048, sign: true }); + data.append(FP16x16 { mag: 941056, sign: true }); + data.append(FP16x16 { mag: 1766400, sign: true }); + data.append(FP16x16 { mag: 1608704, sign: false }); + data.append(FP16x16 { mag: 890368, sign: false }); + data.append(FP16x16 { mag: 236800, sign: true }); + data.append(FP16x16 { mag: 1204224, sign: false }); + data.append(FP16x16 { mag: 554496, sign: true }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/global_average_pool_fp16x16_4D/output_0.cairo b/tests/nodes/global_average_pool_fp16x16_4D/output_0.cairo new file mode 100644 index 000000000..541e2d7a4 --- /dev/null +++ b/tests/nodes/global_average_pool_fp16x16_4D/output_0.cairo @@ -0,0 +1,21 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP16x16Tensor, FP16x16TensorAdd}; +use orion::numbers::{FixedTrait, FP16x16}; + +fn output_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(3); + shape.append(2); + shape.append(1); + shape.append(1); + + let mut data = ArrayTrait::new(); + data.append(FP16x16 { mag: 168320, sign: true }); + data.append(FP16x16 { mag: 134656, sign: false }); + data.append(FP16x16 { mag: 800768, sign: false }); + data.append(FP16x16 { mag: 368896, sign: true }); + data.append(FP16x16 { mag: 523264, sign: true }); + data.append(FP16x16 { mag: 190976, sign: false }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/global_average_pool_fp8x23_2D.cairo b/tests/nodes/global_average_pool_fp8x23_2D.cairo new file mode 100644 index 000000000..cca162dd5 --- /dev/null +++ b/tests/nodes/global_average_pool_fp8x23_2D.cairo @@ -0,0 +1,20 @@ +mod input_0; +mod output_0; + + +use orion::operators::tensor::FP8x23TensorPartialEq; +use orion::operators::nn::NNTrait; +use orion::numbers::FixedTrait; +use orion::operators::nn::FP8x23NN; +use orion::utils::{assert_eq, assert_seq_eq}; + +#[test] +#[available_gas(2000000000)] +fn test_global_average_pool_fp8x23_2D() { + let input_0 = input_0::input_0(); + let z_0 = output_0::output_0(); + + let y_0 = NNTrait::global_average_pool(@input_0); + + assert_eq(y_0, z_0); +} diff --git a/tests/nodes/global_average_pool_fp8x23_2D/input_0.cairo b/tests/nodes/global_average_pool_fp8x23_2D/input_0.cairo new file mode 100644 index 000000000..42bd240b7 --- /dev/null +++ b/tests/nodes/global_average_pool_fp8x23_2D/input_0.cairo @@ -0,0 +1,21 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP8x23Tensor, FP8x23TensorAdd}; +use orion::numbers::{FixedTrait, FP8x23}; + +fn input_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(2); + shape.append(4); + + let mut data = ArrayTrait::new(); + data.append(FP8x23 { mag: 135376663, sign: true }); + data.append(FP8x23 { mag: 141510741, sign: true }); + data.append(FP8x23 { mag: 178081079, sign: true }); + data.append(FP8x23 { mag: 60683532, sign: true }); + data.append(FP8x23 { mag: 141357154, sign: false }); + data.append(FP8x23 { mag: 113733774, sign: false }); + data.append(FP8x23 { mag: 242689028, sign: true }); + data.append(FP8x23 { mag: 105029373, sign: true }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/global_average_pool_fp8x23_2D/output_0.cairo b/tests/nodes/global_average_pool_fp8x23_2D/output_0.cairo new file mode 100644 index 000000000..05867a280 --- /dev/null +++ b/tests/nodes/global_average_pool_fp8x23_2D/output_0.cairo @@ -0,0 +1,21 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP8x23Tensor, FP8x23TensorAdd}; +use orion::numbers::{FixedTrait, FP8x23}; + +fn output_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(2); + shape.append(4); + + let mut data = ArrayTrait::new(); + data.append(FP8x23 { mag: 135376663, sign: true }); + data.append(FP8x23 { mag: 141510741, sign: true }); + data.append(FP8x23 { mag: 178081079, sign: true }); + data.append(FP8x23 { mag: 60683532, sign: true }); + data.append(FP8x23 { mag: 141357154, sign: false }); + data.append(FP8x23 { mag: 113733774, sign: false }); + data.append(FP8x23 { mag: 242689028, sign: true }); + data.append(FP8x23 { mag: 105029373, sign: true }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/global_average_pool_fp8x23_3D.cairo b/tests/nodes/global_average_pool_fp8x23_3D.cairo new file mode 100644 index 000000000..1cdb8ed6a --- /dev/null +++ b/tests/nodes/global_average_pool_fp8x23_3D.cairo @@ -0,0 +1,20 @@ +mod input_0; +mod output_0; + + +use orion::operators::tensor::FP8x23TensorPartialEq; +use orion::operators::nn::NNTrait; +use orion::numbers::FixedTrait; +use orion::operators::nn::FP8x23NN; +use orion::utils::{assert_eq, assert_seq_eq}; + +#[test] +#[available_gas(2000000000)] +fn test_global_average_pool_fp8x23_3D() { + let input_0 = input_0::input_0(); + let z_0 = output_0::output_0(); + + let y_0 = NNTrait::global_average_pool(@input_0); + + assert_eq(y_0, z_0); +} diff --git a/tests/nodes/global_average_pool_fp8x23_3D/input_0.cairo b/tests/nodes/global_average_pool_fp8x23_3D/input_0.cairo new file mode 100644 index 000000000..af43d9ca9 --- /dev/null +++ b/tests/nodes/global_average_pool_fp8x23_3D/input_0.cairo @@ -0,0 +1,30 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP8x23Tensor, FP8x23TensorAdd}; +use orion::numbers::{FixedTrait, FP8x23}; + +fn input_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(2); + shape.append(4); + shape.append(2); + + let mut data = ArrayTrait::new(); + data.append(FP8x23 { mag: 57225710, sign: true }); + data.append(FP8x23 { mag: 206036055, sign: false }); + data.append(FP8x23 { mag: 52167302, sign: false }); + data.append(FP8x23 { mag: 70273911, sign: true }); + data.append(FP8x23 { mag: 116495619, sign: false }); + data.append(FP8x23 { mag: 170231765, sign: true }); + data.append(FP8x23 { mag: 200075176, sign: true }); + data.append(FP8x23 { mag: 51815704, sign: false }); + data.append(FP8x23 { mag: 62353036, sign: true }); + data.append(FP8x23 { mag: 105106308, sign: false }); + data.append(FP8x23 { mag: 228441912, sign: false }); + data.append(FP8x23 { mag: 183099120, sign: true }); + data.append(FP8x23 { mag: 118036599, sign: false }); + data.append(FP8x23 { mag: 137653027, sign: false }); + data.append(FP8x23 { mag: 232603284, sign: false }); + data.append(FP8x23 { mag: 32598207, sign: false }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/global_average_pool_fp8x23_3D/output_0.cairo b/tests/nodes/global_average_pool_fp8x23_3D/output_0.cairo new file mode 100644 index 000000000..0f2fea811 --- /dev/null +++ b/tests/nodes/global_average_pool_fp8x23_3D/output_0.cairo @@ -0,0 +1,22 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP8x23Tensor, FP8x23TensorAdd}; +use orion::numbers::{FixedTrait, FP8x23}; + +fn output_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(2); + shape.append(4); + shape.append(1); + + let mut data = ArrayTrait::new(); + data.append(FP8x23 { mag: 74405172, sign: false }); + data.append(FP8x23 { mag: 9053304, sign: true }); + data.append(FP8x23 { mag: 26868073, sign: true }); + data.append(FP8x23 { mag: 74129735, sign: true }); + data.append(FP8x23 { mag: 21376636, sign: false }); + data.append(FP8x23 { mag: 22671396, sign: false }); + data.append(FP8x23 { mag: 127844813, sign: false }); + data.append(FP8x23 { mag: 132600746, sign: false }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/global_average_pool_fp8x23_4D.cairo b/tests/nodes/global_average_pool_fp8x23_4D.cairo new file mode 100644 index 000000000..a6eb4a75e --- /dev/null +++ b/tests/nodes/global_average_pool_fp8x23_4D.cairo @@ -0,0 +1,20 @@ +mod input_0; +mod output_0; + + +use orion::operators::tensor::FP8x23TensorPartialEq; +use orion::operators::nn::NNTrait; +use orion::numbers::FixedTrait; +use orion::operators::nn::FP8x23NN; +use orion::utils::{assert_eq, assert_seq_eq}; + +#[test] +#[available_gas(2000000000)] +fn test_global_average_pool_fp8x23_4D() { + let input_0 = input_0::input_0(); + let z_0 = output_0::output_0(); + + let y_0 = NNTrait::global_average_pool(@input_0); + + assert_eq(y_0, z_0); +} diff --git a/tests/nodes/global_average_pool_fp8x23_4D/input_0.cairo b/tests/nodes/global_average_pool_fp8x23_4D/input_0.cairo new file mode 100644 index 000000000..1db58c895 --- /dev/null +++ b/tests/nodes/global_average_pool_fp8x23_4D/input_0.cairo @@ -0,0 +1,47 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP8x23Tensor, FP8x23TensorAdd}; +use orion::numbers::{FixedTrait, FP8x23}; + +fn input_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(2); + shape.append(4); + shape.append(2); + shape.append(2); + + let mut data = ArrayTrait::new(); + data.append(FP8x23 { mag: 85392644, sign: false }); + data.append(FP8x23 { mag: 61594092, sign: false }); + data.append(FP8x23 { mag: 163676643, sign: true }); + data.append(FP8x23 { mag: 180530738, sign: false }); + data.append(FP8x23 { mag: 168048412, sign: true }); + data.append(FP8x23 { mag: 5915510, sign: false }); + data.append(FP8x23 { mag: 9047009, sign: true }); + data.append(FP8x23 { mag: 46030420, sign: false }); + data.append(FP8x23 { mag: 184797857, sign: false }); + data.append(FP8x23 { mag: 129370611, sign: false }); + data.append(FP8x23 { mag: 174006060, sign: true }); + data.append(FP8x23 { mag: 162252480, sign: false }); + data.append(FP8x23 { mag: 139240444, sign: true }); + data.append(FP8x23 { mag: 168836878, sign: true }); + data.append(FP8x23 { mag: 246913333, sign: true }); + data.append(FP8x23 { mag: 1047194, sign: true }); + data.append(FP8x23 { mag: 238599466, sign: true }); + data.append(FP8x23 { mag: 216763643, sign: true }); + data.append(FP8x23 { mag: 40581779, sign: true }); + data.append(FP8x23 { mag: 209811161, sign: true }); + data.append(FP8x23 { mag: 250078311, sign: false }); + data.append(FP8x23 { mag: 31811183, sign: true }); + data.append(FP8x23 { mag: 36411415, sign: true }); + data.append(FP8x23 { mag: 107986324, sign: false }); + data.append(FP8x23 { mag: 69727339, sign: false }); + data.append(FP8x23 { mag: 223159880, sign: true }); + data.append(FP8x23 { mag: 184932087, sign: true }); + data.append(FP8x23 { mag: 118617436, sign: false }); + data.append(FP8x23 { mag: 134825391, sign: true }); + data.append(FP8x23 { mag: 217861279, sign: false }); + data.append(FP8x23 { mag: 199069387, sign: false }); + data.append(FP8x23 { mag: 192925915, sign: true }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/global_average_pool_fp8x23_4D/output_0.cairo b/tests/nodes/global_average_pool_fp8x23_4D/output_0.cairo new file mode 100644 index 000000000..791b784aa --- /dev/null +++ b/tests/nodes/global_average_pool_fp8x23_4D/output_0.cairo @@ -0,0 +1,23 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP8x23Tensor, FP8x23TensorAdd}; +use orion::numbers::{FixedTrait, FP8x23}; + +fn output_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(2); + shape.append(4); + shape.append(1); + shape.append(1); + + let mut data = ArrayTrait::new(); + data.append(FP8x23 { mag: 40960207, sign: false }); + data.append(FP8x23 { mag: 31287372, sign: true }); + data.append(FP8x23 { mag: 75603722, sign: false }); + data.append(FP8x23 { mag: 139009462, sign: true }); + data.append(FP8x23 { mag: 176439012, sign: true }); + data.append(FP8x23 { mag: 72460509, sign: false }); + data.append(FP8x23 { mag: 54936798, sign: true }); + data.append(FP8x23 { mag: 22294840, sign: false }); + TensorTrait::new(shape.span(), data.span()) +} From 04157329be8cb73b348ec442213b7be5bb958d75 Mon Sep 17 00:00:00 2001 From: Canance Chan Date: Thu, 14 Mar 2024 02:59:19 +0000 Subject: [PATCH 2/2] Feat: LRN --- .../operators/neural-network/README.md | 2 +- .../operators/neural-network/nn.lrn.md | 84 ++++++++++++++ nodegen/node/lrn.py | 69 +++++++++++ src/operators/nn/core.cairo | 108 ++++++++++-------- src/operators/nn/functional.cairo | 2 +- src/operators/nn/functional/lrn.cairo | 82 +++++++++++++ .../nn/implementations/nn_fp16x16.cairo | 4 +- .../nn/implementations/nn_fp32x32.cairo | 4 +- .../nn/implementations/nn_fp64x64.cairo | 4 +- .../nn/implementations/nn_fp8x23.cairo | 4 +- src/operators/nn/implementations/nn_i32.cairo | 2 +- src/operators/nn/implementations/nn_i8.cairo | 2 +- src/operators/nn/implementations/nn_u32.cairo | 2 +- tests/nodes.cairo | 9 +- tests/nodes/lrn_fp16x16.cairo | 20 ++++ tests/nodes/lrn_fp16x16/input_0.cairo | 51 +++++++++ tests/nodes/lrn_fp16x16/output_0.cairo | 51 +++++++++ tests/nodes/lrn_fp8x23.cairo | 20 ++++ tests/nodes/lrn_fp8x23/input_0.cairo | 47 ++++++++ tests/nodes/lrn_fp8x23/output_0.cairo | 47 ++++++++ 20 files changed, 544 insertions(+), 70 deletions(-) create mode 100644 docs/framework/operators/neural-network/nn.lrn.md create mode 100644 nodegen/node/lrn.py create mode 100644 src/operators/nn/functional/lrn.cairo create mode 100644 tests/nodes/lrn_fp16x16.cairo create mode 100644 tests/nodes/lrn_fp16x16/input_0.cairo create mode 100644 tests/nodes/lrn_fp16x16/output_0.cairo create mode 100644 tests/nodes/lrn_fp8x23.cairo create mode 100644 tests/nodes/lrn_fp8x23/input_0.cairo create mode 100644 tests/nodes/lrn_fp8x23/output_0.cairo diff --git a/docs/framework/operators/neural-network/README.md b/docs/framework/operators/neural-network/README.md index 1dd628070..1b5265a02 100644 --- a/docs/framework/operators/neural-network/README.md +++ b/docs/framework/operators/neural-network/README.md @@ -39,5 +39,5 @@ Orion supports currently these `NN` types. | [`nn.col2im`](nn.col2im.md) | Rearranges column blocks back into a multidimensional image | | [`nn.conv_transpose`](nn.conv\_transpose.md) | Performs the convolution transpose of the input data tensor and weight tensor. | | [`nn.conv`](nn.conv.md) | Performs the convolution of the input data tensor and weight tensor. | -| [`nn.global_average_pool`](nn.global\_average\_pool.md) | GlobalAveragePool consumes an input tensor X and applies average pooling across the values in the same channel. | +| [`nn.lrn`](nn.lrn.md) | Local Response Normalization proposed in the AlexNet paper. It normalizes over local input regions. The local region is defined across the channels. | diff --git a/docs/framework/operators/neural-network/nn.lrn.md b/docs/framework/operators/neural-network/nn.lrn.md new file mode 100644 index 000000000..c9f5b2927 --- /dev/null +++ b/docs/framework/operators/neural-network/nn.lrn.md @@ -0,0 +1,84 @@ +# NNTrait::lrn + +```rust + fn lrn(tensor: @Tensor, size: usize, alpha: Option, beta: Option, bias: Option) -> Tensor; +``` + +Local Response Normalization proposed in the AlexNet paper. It normalizes over local input regions. The local region is defined across the channels. + +## Args + +* `tensor`(`@Tensor`) - Input data tensor from the previous operator; dimensions for image case are (N x C x H x W), where N is the batch size, C is the number of channels, and H and W are the height and the width of the data. +* `size`(usize) - The number of channels to sum over. +* `alpha`(Option) - Scaling parameter. +* `beta`(Option) - The exponent. +* `bias`(Option) + +## Returns + +* A `Tensor` with the same shape as the input tensor. + +## Examples + +```rust +use orion::operators::tensor::{FP16x16Tensor, FP16x16TensorAdd}; +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::utils::{assert_eq, assert_seq_eq}; +use orion::operators::tensor::FP16x16TensorPartialEq; +use orion::numbers::{FixedTrait, FP16x16}; +use orion::operators::nn::NNTrait; +use orion::operators::nn::FP16x16NN; + +fn example() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(3); + shape.append(2); + shape.append(2); + shape.append(3); + + let mut data = ArrayTrait::new(); + data.append(FP16x16 { mag: 1400832, sign: true }); + data.append(FP16x16 { mag: 85184, sign: true }); + data.append(FP16x16 { mag: 399616, sign: false }); + data.append(FP16x16 { mag: 1828864, sign: false }); + data.append(FP16x16 { mag: 962560, sign: true }); + data.append(FP16x16 { mag: 1190912, sign: false }); + data.append(FP16x16 { mag: 1355776, sign: true }); + data.append(FP16x16 { mag: 946176, sign: false }); + data.append(FP16x16 { mag: 502528, sign: true }); + data.append(FP16x16 { mag: 85824, sign: false }); + data.append(FP16x16 { mag: 1333248, sign: false }); + data.append(FP16x16 { mag: 39200, sign: true }); + data.append(FP16x16 { mag: 712192, sign: false }); + data.append(FP16x16 { mag: 1497088, sign: false }); + data.append(FP16x16 { mag: 1521664, sign: false }); + data.append(FP16x16 { mag: 606720, sign: true }); + data.append(FP16x16 { mag: 848384, sign: false }); + data.append(FP16x16 { mag: 1732608, sign: true }); + data.append(FP16x16 { mag: 1158144, sign: true }); + data.append(FP16x16 { mag: 1806336, sign: false }); + data.append(FP16x16 { mag: 935424, sign: true }); + data.append(FP16x16 { mag: 1106944, sign: true }); + data.append(FP16x16 { mag: 1180672, sign: true }); + data.append(FP16x16 { mag: 1509376, sign: true }); + data.append(FP16x16 { mag: 856064, sign: false }); + data.append(FP16x16 { mag: 1841152, sign: false }); + data.append(FP16x16 { mag: 75008, sign: false }); + data.append(FP16x16 { mag: 19504, sign: true }); + data.append(FP16x16 { mag: 1326080, sign: false }); + data.append(FP16x16 { mag: 1423360, sign: true }); + data.append(FP16x16 { mag: 1258496, sign: true }); + data.append(FP16x16 { mag: 671232, sign: true }); + data.append(FP16x16 { mag: 548352, sign: false }); + data.append(FP16x16 { mag: 797696, sign: true }); + data.append(FP16x16 { mag: 1245184, sign: true }); + data.append(FP16x16 { mag: 1404928, sign: false }); + let tensor1 = TensorTrait::new(shape.span(), data.span()); + + return NNTrait::lrn(@tensor1, 3, Option::Some(FP16x16 { mag: 13, sign: false }),Option::Some(FP16x16 { mag: 32768, sign: false }),Option::Some(FP16x16 { mag: 131072, sign: false })); +} +>>> [FP16x16 { mag: 964915, sign: true }, FP16x16 { mag: 60080, sign: true }, FP16x16 { mag: 282570, sign: false }, FP16x16 { mag: 1276847, sign: false }, FP16x16 { mag: 680331, sign: true }, FP16x16 { mag: 838194, sign: false }, FP16x16 { mag: 933879, sign: true }, FP16x16 { mag: 667340, sign: false }, FP16x16 { mag: 355340, sign: true }, FP16x16 { mag: 59919, sign: false }, FP16x16 { mag: 942330, sign: false }, FP16x16 { mag: 27589, sign: true }, FP16x16 { mag: 503258, sign: false }, + FP16x16 { mag: 1019961, sign: false }, FP16x16 { mag: 1074737, sign: false }, FP16x16 { mag: 424644, sign: true }, FP16x16 { mag: 599690, sign: false }, FP16x16 { mag: 1182030, sign: true }, FP16x16 { mag: 818383, sign: true }, FP16x16 { mag: 1230651, sign: false }, FP16x16 { mag: 660681, sign: true }, FP16x16 { mag: 774752, sign: true }, FP16x16 { mag: 834572, sign: true }, FP16x16 { mag: 1029735, sign: true }, FP16x16 { mag: 605008, sign: false }, FP16x16 { mag: 1295696, sign: false }, + FP16x16 { mag: 52967, sign: false }, FP16x16 { mag: 13759, sign: true }, FP16x16 { mag: 937699, sign: false }, FP16x16 { mag: 1006488, sign: true }, FP16x16 { mag: 889419, sign: true }, FP16x16 { mag: 472374, sign: true }, FP16x16 { mag: 387220, sign: false }, FP16x16 { mag: 562768, sign: true }, FP16x16 { mag: 880496, sign: true }, FP16x16 { mag: 993454, sign: false }] +``` \ No newline at end of file diff --git a/nodegen/node/lrn.py b/nodegen/node/lrn.py new file mode 100644 index 000000000..9edc9ab96 --- /dev/null +++ b/nodegen/node/lrn.py @@ -0,0 +1,69 @@ +import numpy as np +from nodegen.node import RunAll +import math +from ..helpers import make_test, to_fp, Tensor, Dtype, FixedImpl, Trait + +def lrn(x, size, alpha=0.0001, beta=0.75, bias=1.0): # type: ignore + if len(x.shape) != 4: + raise RuntimeError( + f"LRN only applies on 4D tensors but shape is {x.shape!r}." + ) + square_sum = np.zeros(x.shape).astype(x.dtype) + minc = x.shape[1] + c1 = int(math.floor((size - 1) / 2)) + c2 = int(math.ceil((size - 1) / 2)) + 1 + for c in range(x.shape[1]): + begin = max(0, c - c1) + end = min(minc, c + c2) + square_sum[:, c, :, :] = np.sum(x[:, begin:end, :, :] ** 2, axis=1) + y = x / ((bias + (alpha / size) * square_sum) ** beta) + return y.astype(x.dtype) + +def get_data_statement(data: np.ndarray, dtype: Dtype) -> list[str]: + match dtype: + case Dtype.FP8x23: + return ["Option::Some(FP8x23 { "+f"mag: {abs(int(x))}, sign: {str(x < 0).lower()} "+"})" for x in data.flatten()] + case Dtype.FP16x16: + return ["Option::Some(FP16x16 { "+f"mag: {abs(int(x))}, sign: {str(x < 0).lower()} "+"})" for x in data.flatten()] + +class Lrn(RunAll): + @staticmethod + def fp8x23(): + alpha = 0.0002 + beta = 0.5 + bias = 2.0 + args = [alpha, beta, bias] + args_str = get_data_statement(to_fp(np.array(args).flatten(), FixedImpl.FP8x23), Dtype.FP8x23) + nsize = 3 + x = np.random.uniform(-30, 30, (2, 4, 2, 2)).astype(np.float64) + y = lrn(x, nsize, *args) + + x = Tensor(Dtype.FP8x23, x.shape, to_fp( + x.flatten(), FixedImpl.FP8x23)) + y = Tensor(Dtype.FP8x23, y.shape, to_fp( + y.flatten(), FixedImpl.FP8x23)) + + name = "lrn_fp8x23" + make_test([x], y, f"NNTrait::lrn(@input_0, {nsize}, {','.join(args_str)})", + name, Trait.NN) + + @staticmethod + def fp16x16(): + alpha = 0.0002 + beta = 0.5 + bias = 2.0 + args = [alpha, beta, bias] + args_str = get_data_statement(to_fp(np.array(args).flatten(), FixedImpl.FP16x16), Dtype.FP16x16) + nsize = 3 + x = np.random.uniform(-30, 30, (3, 2, 2, 3)).astype(np.float16) + y = lrn(x, nsize, *args) + + x = Tensor(Dtype.FP16x16, x.shape, to_fp( + x.flatten(), FixedImpl.FP16x16)) + y = Tensor(Dtype.FP16x16, y.shape, to_fp( + y.flatten(), FixedImpl.FP16x16)) + + name = "lrn_fp16x16" + make_test([x], y, f"NNTrait::lrn(@input_0, {nsize}, {','.join(args_str)})", + name, Trait.NN) + \ No newline at end of file diff --git a/src/operators/nn/core.cairo b/src/operators/nn/core.cairo index 1b75f7ccb..13a9a56c1 100644 --- a/src/operators/nn/core.cairo +++ b/src/operators/nn/core.cairo @@ -18,7 +18,7 @@ use orion::operators::tensor::core::Tensor; /// col2im - Rearranges column blocks back into a multidimensional image /// conv_transpose - Performs the convolution transpose of the input data tensor and weight tensor. /// conv - Performs the convolution of the input data tensor and weight tensor. -/// global_average_pool - GlobalAveragePool consumes an input tensor X and applies average pooling across the values in the same channel. +/// lrn - Local Response Normalization proposed in the AlexNet paper. It normalizes over local input regions. The local region is defined across the channels. trait NNTrait { /// # NNTrait::relu /// @@ -1305,81 +1305,89 @@ trait NNTrait { mode: Option, padding_mode: Option, ) -> Tensor; - /// # NNTrait::global_average_pool + /// # NNTrait::lrn /// /// ```rust - /// fn global_average_pool(tensor: @Tensor) -> Tensor; + /// fn lrn(tensor: @Tensor, size: usize, alpha: Option, beta: Option, bias: Option) -> Tensor; /// ``` /// - /// GlobalAveragePool consumes an input tensor X and applies average pooling across the values in the same channel. - /// This is equivalent to AveragePool with kernel size equal to the spatial dimension of input tensor. + /// Local Response Normalization proposed in the AlexNet paper. It normalizes over local input regions. The local region is defined across the channels. /// /// ## Args /// - /// * `tensor`(`@Tensor`) - Input data tensor from the previous operator; dimensions for image case are (N x C x H x W), where N is the batch size, C is the number of channels, and H and W are the height and the width of the data. For non image case, the dimensions are in the form of (N x C x D1 x D2 ... Dn), where N is the batch size. + /// * `tensor`(`@Tensor`) - Input data tensor from the previous operator; dimensions for image case are (N x C x H x W), where N is the batch size, C is the number of channels, and H and W are the height and the width of the data. + /// * `size`(usize) - The number of channels to sum over. + /// * `alpha`(Option) - Scaling parameter. + /// * `beta`(Option) - The exponent. + /// * `bias`(Option) /// /// ## Returns /// - /// * Output data tensor from pooling across the input tensor. The output tensor has the same rank as the input. The first two dimensions of output shape are the same as the input (N x C), while the other dimensions are all 1. + /// * A `Tensor` with the same shape as the input tensor. /// /// ## Examples /// /// ```rust - /// use orion::operators::tensor::{FP8x23Tensor, FP8x23TensorAdd}; + /// use orion::operators::tensor::{FP16x16Tensor, FP16x16TensorAdd}; /// use core::array::{ArrayTrait, SpanTrait}; /// use orion::operators::tensor::{TensorTrait, Tensor}; /// use orion::utils::{assert_eq, assert_seq_eq}; - /// use orion::operators::tensor::FP8x23TensorPartialEq; - /// use orion::numbers::{FixedTrait, FP8x23}; + /// use orion::operators::tensor::FP16x16TensorPartialEq; + /// use orion::numbers::{FixedTrait, FP16x16}; /// use orion::operators::nn::NNTrait; - /// use orion::operators::nn::FP8x23NN; + /// use orion::operators::nn::FP16x16NN; /// - /// fn example() -> Tensor { + /// fn example() -> Tensor { /// let mut shape = ArrayTrait::::new(); - /// shape.append(2); - /// shape.append(4); + /// shape.append(3); /// shape.append(2); /// shape.append(2); + /// shape.append(3); /// /// let mut data = ArrayTrait::new(); - /// data.append(FP8x23 { mag: 85392644, sign: false }); - /// data.append(FP8x23 { mag: 61594092, sign: false }); - /// data.append(FP8x23 { mag: 163676643, sign: true }); - /// data.append(FP8x23 { mag: 180530738, sign: false }); - /// data.append(FP8x23 { mag: 168048412, sign: true }); - /// data.append(FP8x23 { mag: 5915510, sign: false }); - /// data.append(FP8x23 { mag: 9047009, sign: true }); - /// data.append(FP8x23 { mag: 46030420, sign: false }); - /// data.append(FP8x23 { mag: 184797857, sign: false }); - /// data.append(FP8x23 { mag: 129370611, sign: false }); - /// data.append(FP8x23 { mag: 174006060, sign: true }); - /// data.append(FP8x23 { mag: 162252480, sign: false }); - /// data.append(FP8x23 { mag: 139240444, sign: true }); - /// data.append(FP8x23 { mag: 168836878, sign: true }); - /// data.append(FP8x23 { mag: 246913333, sign: true }); - /// data.append(FP8x23 { mag: 1047194, sign: true }); - /// data.append(FP8x23 { mag: 238599466, sign: true }); - /// data.append(FP8x23 { mag: 216763643, sign: true }); - /// data.append(FP8x23 { mag: 40581779, sign: true }); - /// data.append(FP8x23 { mag: 209811161, sign: true }); - /// data.append(FP8x23 { mag: 250078311, sign: false }); - /// data.append(FP8x23 { mag: 31811183, sign: true }); - /// data.append(FP8x23 { mag: 36411415, sign: true }); - /// data.append(FP8x23 { mag: 107986324, sign: false }); - /// data.append(FP8x23 { mag: 69727339, sign: false }); - /// data.append(FP8x23 { mag: 223159880, sign: true }); - /// data.append(FP8x23 { mag: 184932087, sign: true }); - /// data.append(FP8x23 { mag: 118617436, sign: false }); - /// data.append(FP8x23 { mag: 134825391, sign: true }); - /// data.append(FP8x23 { mag: 217861279, sign: false }); - /// data.append(FP8x23 { mag: 199069387, sign: false }); - /// data.append(FP8x23 { mag: 192925915, sign: true }); + /// data.append(FP16x16 { mag: 1400832, sign: true }); + /// data.append(FP16x16 { mag: 85184, sign: true }); + /// data.append(FP16x16 { mag: 399616, sign: false }); + /// data.append(FP16x16 { mag: 1828864, sign: false }); + /// data.append(FP16x16 { mag: 962560, sign: true }); + /// data.append(FP16x16 { mag: 1190912, sign: false }); + /// data.append(FP16x16 { mag: 1355776, sign: true }); + /// data.append(FP16x16 { mag: 946176, sign: false }); + /// data.append(FP16x16 { mag: 502528, sign: true }); + /// data.append(FP16x16 { mag: 85824, sign: false }); + /// data.append(FP16x16 { mag: 1333248, sign: false }); + /// data.append(FP16x16 { mag: 39200, sign: true }); + /// data.append(FP16x16 { mag: 712192, sign: false }); + /// data.append(FP16x16 { mag: 1497088, sign: false }); + /// data.append(FP16x16 { mag: 1521664, sign: false }); + /// data.append(FP16x16 { mag: 606720, sign: true }); + /// data.append(FP16x16 { mag: 848384, sign: false }); + /// data.append(FP16x16 { mag: 1732608, sign: true }); + /// data.append(FP16x16 { mag: 1158144, sign: true }); + /// data.append(FP16x16 { mag: 1806336, sign: false }); + /// data.append(FP16x16 { mag: 935424, sign: true }); + /// data.append(FP16x16 { mag: 1106944, sign: true }); + /// data.append(FP16x16 { mag: 1180672, sign: true }); + /// data.append(FP16x16 { mag: 1509376, sign: true }); + /// data.append(FP16x16 { mag: 856064, sign: false }); + /// data.append(FP16x16 { mag: 1841152, sign: false }); + /// data.append(FP16x16 { mag: 75008, sign: false }); + /// data.append(FP16x16 { mag: 19504, sign: true }); + /// data.append(FP16x16 { mag: 1326080, sign: false }); + /// data.append(FP16x16 { mag: 1423360, sign: true }); + /// data.append(FP16x16 { mag: 1258496, sign: true }); + /// data.append(FP16x16 { mag: 671232, sign: true }); + /// data.append(FP16x16 { mag: 548352, sign: false }); + /// data.append(FP16x16 { mag: 797696, sign: true }); + /// data.append(FP16x16 { mag: 1245184, sign: true }); + /// data.append(FP16x16 { mag: 1404928, sign: false }); /// let tensor1 = TensorTrait::new(shape.span(), data.span()); /// - /// return NNTrait::global_average_pool(@tensor1); + /// return NNTrait::lrn(@tensor1, 3, Option::Some(FP16x16 { mag: 13, sign: false }),Option::Some(FP16x16 { mag: 32768, sign: false }),Option::Some(FP16x16 { mag: 131072, sign: false })); /// } - /// >>> [{ mag: 40960207, sign: true } { mag: 31287372, sign: false } { mag: 75603722, sign: true } { mag: 139009462, sign: false } { mag: 176439012, sign: false } { mag: 72460509, sign: true } { mag: 54936798, sign: false } { mag: 22294840, sign: true } ] + /// >>> [FP16x16 { mag: 964915, sign: true }, FP16x16 { mag: 60080, sign: true }, FP16x16 { mag: 282570, sign: false }, FP16x16 { mag: 1276847, sign: false }, FP16x16 { mag: 680331, sign: true }, FP16x16 { mag: 838194, sign: false }, FP16x16 { mag: 933879, sign: true }, FP16x16 { mag: 667340, sign: false }, FP16x16 { mag: 355340, sign: true }, FP16x16 { mag: 59919, sign: false }, FP16x16 { mag: 942330, sign: false }, FP16x16 { mag: 27589, sign: true }, FP16x16 { mag: 503258, sign: false }, + /// FP16x16 { mag: 1019961, sign: false }, FP16x16 { mag: 1074737, sign: false }, FP16x16 { mag: 424644, sign: true }, FP16x16 { mag: 599690, sign: false }, FP16x16 { mag: 1182030, sign: true }, FP16x16 { mag: 818383, sign: true }, FP16x16 { mag: 1230651, sign: false }, FP16x16 { mag: 660681, sign: true }, FP16x16 { mag: 774752, sign: true }, FP16x16 { mag: 834572, sign: true }, FP16x16 { mag: 1029735, sign: true }, FP16x16 { mag: 605008, sign: false }, FP16x16 { mag: 1295696, sign: false }, + /// FP16x16 { mag: 52967, sign: false }, FP16x16 { mag: 13759, sign: true }, FP16x16 { mag: 937699, sign: false }, FP16x16 { mag: 1006488, sign: true }, FP16x16 { mag: 889419, sign: true }, FP16x16 { mag: 472374, sign: true }, FP16x16 { mag: 387220, sign: false }, FP16x16 { mag: 562768, sign: true }, FP16x16 { mag: 880496, sign: true }, FP16x16 { mag: 993454, sign: false }] /// ``` - /// - fn global_average_pool(tensor: @Tensor) -> Tensor; + fn lrn(tensor: @Tensor, size: usize, alpha: Option, beta: Option, bias: Option) -> Tensor; } diff --git a/src/operators/nn/functional.cairo b/src/operators/nn/functional.cairo index b1734558e..577e38cd3 100644 --- a/src/operators/nn/functional.cairo +++ b/src/operators/nn/functional.cairo @@ -16,4 +16,4 @@ mod conv_transpose; mod depth_to_space; mod space_to_depth; mod conv; -mod global_average_pool; +mod lrn; diff --git a/src/operators/nn/functional/lrn.cairo b/src/operators/nn/functional/lrn.cairo new file mode 100644 index 000000000..e6f67037e --- /dev/null +++ b/src/operators/nn/functional/lrn.cairo @@ -0,0 +1,82 @@ +use orion::numbers::fixed_point::core::FixedTrait; +use orion::numbers::NumberTrait; +use orion::operators::tensor::core::{Tensor, TensorTrait}; +use orion::operators::tensor::helpers::{reduce_output_shape, len_from_shape, combine_indices}; +use orion::operators::tensor::math::{reduce_sum::accumulate_sum, arithmetic::div_downcast}; + +fn lrn< + T, + MAG, + impl TTensor: TensorTrait, + impl TNumber: NumberTrait, + impl TAdd: Add, + impl TSub: Sub, + impl TMul: Mul, + impl TDiv: Div, + impl TTensorAdd: Add>, + impl TPartialOrd: PartialOrd, + impl TAddEq: AddEq, + impl TCopy: Copy, + impl TDrop: Drop, +>( + tensor: Tensor, size: usize, alpha: Option, beta: Option, bias: Option +) -> Tensor { + assert((tensor.shape).len() >= 2, 'Unexpected shape.'); + + let minc = *(tensor.shape).at(1); + + let zero: T = NumberTrait::zero(); + let one: T = NumberTrait::one(); + let two: T = one + one; + let three: T = two + one; + let ten: T = three.pow(two) + one; + let four: T = two + two; + let n_10000: T = ten.pow(zero - four); + let alpha_v = match alpha { + Option::Some(val) => val, + Option::None => one / n_10000, + }; + let beta_v = match beta { + Option::Some(val) => val, + Option::None => three / (two * two), + }; + let bias_v = match bias { + Option::Some(val) => val, + Option::None => one, + }; + let c1: usize = (size - 1) / 2; + let mut c2: usize = ((size - 1) / 2) + 1; + if (c1 * 2 + 1 != size) { + c2 += 1; + }; + let mut c: usize = 0; + let mut sum_array: Array> = array![]; + while c != minc { + let mut begin = 0; + if c > c1 { + begin = c - c1; + }; + let end = NumberTrait::min(minc, c + c2); + let s = tensor.slice(array![begin].span(), array![end].span(), Option::Some(array![1].span()), Option::Some(array![1].span())); + sum_array.append(s.reduce_sum(axis: 1, keepdims: true)); + c += 1; + }; + let sum_concat = TensorTrait::concat(sum_array.span(), 1); + let len = (tensor.data).len(); + let mut arr: Array = array![]; + let mut i: usize = 0; + let mut size_t: T = zero; + while i != size { + size_t = size_t + one; + i += 1; + }; + i = 0; + while i != len { + let v1 = *(sum_concat.data).at(i); + let v2 = *(tensor.data).at(i); + let r = v2 / ((bias_v + (alpha_v / size_t) * v1 * v1).pow(beta_v)); + arr.append(r); + i += 1; + }; + TensorTrait::::new(tensor.shape, arr.span()) +} diff --git a/src/operators/nn/implementations/nn_fp16x16.cairo b/src/operators/nn/implementations/nn_fp16x16.cairo index 758dc2331..823ca2960 100644 --- a/src/operators/nn/implementations/nn_fp16x16.cairo +++ b/src/operators/nn/implementations/nn_fp16x16.cairo @@ -144,7 +144,7 @@ impl FP16x16NN of NNTrait { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } - fn global_average_pool(tensor: @Tensor) -> Tensor { - functional::global_average_pool::global_average_pool(*tensor) + fn lrn(tensor: @Tensor, size: usize, alpha: Option, beta: Option, bias: Option) -> Tensor { + functional::lrn::lrn(*tensor, size, alpha, beta, bias) } } diff --git a/src/operators/nn/implementations/nn_fp32x32.cairo b/src/operators/nn/implementations/nn_fp32x32.cairo index b8ead0f91..f4d79f6dd 100644 --- a/src/operators/nn/implementations/nn_fp32x32.cairo +++ b/src/operators/nn/implementations/nn_fp32x32.cairo @@ -138,7 +138,7 @@ impl FP32x32NN of NNTrait { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } - fn global_average_pool(tensor: @Tensor) -> Tensor { - functional::global_average_pool::global_average_pool(*tensor) + fn lrn(tensor: @Tensor, size: usize, alpha: Option, beta: Option, bias: Option) -> Tensor { + functional::lrn::lrn(*tensor, size, alpha, beta, bias) } } diff --git a/src/operators/nn/implementations/nn_fp64x64.cairo b/src/operators/nn/implementations/nn_fp64x64.cairo index 2d857d7d7..b2ed601c6 100644 --- a/src/operators/nn/implementations/nn_fp64x64.cairo +++ b/src/operators/nn/implementations/nn_fp64x64.cairo @@ -138,7 +138,7 @@ impl FP64x64NN of NNTrait { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } - fn global_average_pool(tensor: @Tensor) -> Tensor { - functional::global_average_pool::global_average_pool(*tensor) + fn lrn(tensor: @Tensor, size: usize, alpha: Option, beta: Option, bias: Option) -> Tensor { + functional::lrn::lrn(*tensor, size, alpha, beta, bias) } } diff --git a/src/operators/nn/implementations/nn_fp8x23.cairo b/src/operators/nn/implementations/nn_fp8x23.cairo index e49774954..3c90d2201 100644 --- a/src/operators/nn/implementations/nn_fp8x23.cairo +++ b/src/operators/nn/implementations/nn_fp8x23.cairo @@ -140,7 +140,7 @@ impl FP8x23NN of NNTrait { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } - fn global_average_pool(tensor: @Tensor) -> Tensor { - functional::global_average_pool::global_average_pool(*tensor) + fn lrn(tensor: @Tensor, size: usize, alpha: Option, beta: Option, bias: Option) -> Tensor { + functional::lrn::lrn(*tensor, size, alpha, beta, bias) } } diff --git a/src/operators/nn/implementations/nn_i32.cairo b/src/operators/nn/implementations/nn_i32.cairo index def7aa7d8..e4a1b7ea5 100644 --- a/src/operators/nn/implementations/nn_i32.cairo +++ b/src/operators/nn/implementations/nn_i32.cairo @@ -131,7 +131,7 @@ impl I32NN of NNTrait { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } - fn global_average_pool(tensor: @Tensor) -> Tensor { + fn lrn(tensor: @Tensor, size: usize, alpha: Option, beta: Option, bias: Option) -> Tensor { panic(array!['not supported!']) } } diff --git a/src/operators/nn/implementations/nn_i8.cairo b/src/operators/nn/implementations/nn_i8.cairo index d1771a631..acc8eba99 100644 --- a/src/operators/nn/implementations/nn_i8.cairo +++ b/src/operators/nn/implementations/nn_i8.cairo @@ -131,7 +131,7 @@ impl I8NN of NNTrait { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } - fn global_average_pool(tensor: @Tensor) -> Tensor { + fn lrn(tensor: @Tensor, size: usize, alpha: Option, beta: Option, bias: Option) -> Tensor { panic(array!['not supported!']) } } diff --git a/src/operators/nn/implementations/nn_u32.cairo b/src/operators/nn/implementations/nn_u32.cairo index 67072c4f0..6f55c6ea4 100644 --- a/src/operators/nn/implementations/nn_u32.cairo +++ b/src/operators/nn/implementations/nn_u32.cairo @@ -131,7 +131,7 @@ impl U32NN of NNTrait { functional::conv::conv(X, W, B, auto_pad, dilations, group, kernel_shape, pads, strides) } - fn global_average_pool(tensor: @Tensor) -> Tensor { + fn lrn(tensor: @Tensor, size: usize, alpha: Option, beta: Option, bias: Option) -> Tensor { panic(array!['not supported!']) } } diff --git a/tests/nodes.cairo b/tests/nodes.cairo index bc8a9ea7d..8748d444e 100644 --- a/tests/nodes.cairo +++ b/tests/nodes.cairo @@ -1047,10 +1047,5 @@ mod label_encoder_fp8x23_default; mod label_encoder_i8_default; mod label_encoder_i32_default; mod label_encoder_u32_default; -mod global_average_pool_fp16x16_2D; -mod global_average_pool_fp16x16_3D; -mod global_average_pool_fp16x16_4D; -mod global_average_pool_fp8x23_2D; -mod global_average_pool_fp8x23_3D; -mod global_average_pool_fp8x23_4D; - +mod lrn_fp8x23; +mod lrn_fp16x16; diff --git a/tests/nodes/lrn_fp16x16.cairo b/tests/nodes/lrn_fp16x16.cairo new file mode 100644 index 000000000..7869770b6 --- /dev/null +++ b/tests/nodes/lrn_fp16x16.cairo @@ -0,0 +1,20 @@ +mod input_0; +mod output_0; + + +use orion::operators::tensor::FP16x16TensorPartialEq; +use orion::operators::nn::NNTrait; +use orion::utils::{assert_eq, assert_seq_eq}; +use orion::operators::nn::FP16x16NN; +use orion::numbers::{FixedTrait, FP16x16}; + +#[test] +#[available_gas(2000000000)] +fn test_lrn_fp16x16() { + let input_0 = input_0::input_0(); + let z_0 = output_0::output_0(); + + let y_0 = NNTrait::lrn(@input_0, 3, Option::Some(FP16x16 { mag: 13, sign: false }),Option::Some(FP16x16 { mag: 32768, sign: false }),Option::Some(FP16x16 { mag: 131072, sign: false })); + + assert_eq(y_0, z_0); +} diff --git a/tests/nodes/lrn_fp16x16/input_0.cairo b/tests/nodes/lrn_fp16x16/input_0.cairo new file mode 100644 index 000000000..32380284a --- /dev/null +++ b/tests/nodes/lrn_fp16x16/input_0.cairo @@ -0,0 +1,51 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP16x16Tensor, FP16x16TensorAdd}; +use orion::numbers::{FixedTrait, FP16x16}; + +fn input_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(3); + shape.append(2); + shape.append(2); + shape.append(3); + + let mut data = ArrayTrait::new(); + data.append(FP16x16 { mag: 1400832, sign: true }); + data.append(FP16x16 { mag: 85184, sign: true }); + data.append(FP16x16 { mag: 399616, sign: false }); + data.append(FP16x16 { mag: 1828864, sign: false }); + data.append(FP16x16 { mag: 962560, sign: true }); + data.append(FP16x16 { mag: 1190912, sign: false }); + data.append(FP16x16 { mag: 1355776, sign: true }); + data.append(FP16x16 { mag: 946176, sign: false }); + data.append(FP16x16 { mag: 502528, sign: true }); + data.append(FP16x16 { mag: 85824, sign: false }); + data.append(FP16x16 { mag: 1333248, sign: false }); + data.append(FP16x16 { mag: 39200, sign: true }); + data.append(FP16x16 { mag: 712192, sign: false }); + data.append(FP16x16 { mag: 1497088, sign: false }); + data.append(FP16x16 { mag: 1521664, sign: false }); + data.append(FP16x16 { mag: 606720, sign: true }); + data.append(FP16x16 { mag: 848384, sign: false }); + data.append(FP16x16 { mag: 1732608, sign: true }); + data.append(FP16x16 { mag: 1158144, sign: true }); + data.append(FP16x16 { mag: 1806336, sign: false }); + data.append(FP16x16 { mag: 935424, sign: true }); + data.append(FP16x16 { mag: 1106944, sign: true }); + data.append(FP16x16 { mag: 1180672, sign: true }); + data.append(FP16x16 { mag: 1509376, sign: true }); + data.append(FP16x16 { mag: 856064, sign: false }); + data.append(FP16x16 { mag: 1841152, sign: false }); + data.append(FP16x16 { mag: 75008, sign: false }); + data.append(FP16x16 { mag: 19504, sign: true }); + data.append(FP16x16 { mag: 1326080, sign: false }); + data.append(FP16x16 { mag: 1423360, sign: true }); + data.append(FP16x16 { mag: 1258496, sign: true }); + data.append(FP16x16 { mag: 671232, sign: true }); + data.append(FP16x16 { mag: 548352, sign: false }); + data.append(FP16x16 { mag: 797696, sign: true }); + data.append(FP16x16 { mag: 1245184, sign: true }); + data.append(FP16x16 { mag: 1404928, sign: false }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/lrn_fp16x16/output_0.cairo b/tests/nodes/lrn_fp16x16/output_0.cairo new file mode 100644 index 000000000..81bf7563a --- /dev/null +++ b/tests/nodes/lrn_fp16x16/output_0.cairo @@ -0,0 +1,51 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP16x16Tensor, FP16x16TensorAdd}; +use orion::numbers::{FixedTrait, FP16x16}; + +fn output_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(3); + shape.append(2); + shape.append(2); + shape.append(3); + + let mut data = ArrayTrait::new(); + data.append(FP16x16 { mag: 964915, sign: true }); + data.append(FP16x16 { mag: 60080, sign: true }); + data.append(FP16x16 { mag: 282570, sign: false }); + data.append(FP16x16 { mag: 1276847, sign: false }); + data.append(FP16x16 { mag: 680331, sign: true }); + data.append(FP16x16 { mag: 838194, sign: false }); + data.append(FP16x16 { mag: 933879, sign: true }); + data.append(FP16x16 { mag: 667340, sign: false }); + data.append(FP16x16 { mag: 355340, sign: true }); + data.append(FP16x16 { mag: 59919, sign: false }); + data.append(FP16x16 { mag: 942330, sign: false }); + data.append(FP16x16 { mag: 27589, sign: true }); + data.append(FP16x16 { mag: 503258, sign: false }); + data.append(FP16x16 { mag: 1019961, sign: false }); + data.append(FP16x16 { mag: 1074737, sign: false }); + data.append(FP16x16 { mag: 424644, sign: true }); + data.append(FP16x16 { mag: 599690, sign: false }); + data.append(FP16x16 { mag: 1182030, sign: true }); + data.append(FP16x16 { mag: 818383, sign: true }); + data.append(FP16x16 { mag: 1230651, sign: false }); + data.append(FP16x16 { mag: 660681, sign: true }); + data.append(FP16x16 { mag: 774752, sign: true }); + data.append(FP16x16 { mag: 834572, sign: true }); + data.append(FP16x16 { mag: 1029735, sign: true }); + data.append(FP16x16 { mag: 605008, sign: false }); + data.append(FP16x16 { mag: 1295696, sign: false }); + data.append(FP16x16 { mag: 52967, sign: false }); + data.append(FP16x16 { mag: 13759, sign: true }); + data.append(FP16x16 { mag: 937699, sign: false }); + data.append(FP16x16 { mag: 1006488, sign: true }); + data.append(FP16x16 { mag: 889419, sign: true }); + data.append(FP16x16 { mag: 472374, sign: true }); + data.append(FP16x16 { mag: 387220, sign: false }); + data.append(FP16x16 { mag: 562768, sign: true }); + data.append(FP16x16 { mag: 880496, sign: true }); + data.append(FP16x16 { mag: 993454, sign: false }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/lrn_fp8x23.cairo b/tests/nodes/lrn_fp8x23.cairo new file mode 100644 index 000000000..5b8502c41 --- /dev/null +++ b/tests/nodes/lrn_fp8x23.cairo @@ -0,0 +1,20 @@ +mod input_0; +mod output_0; + + +use orion::operators::nn::FP8x23NN; +use orion::utils::{assert_eq, assert_seq_eq}; +use orion::operators::nn::NNTrait; +use orion::operators::tensor::FP8x23TensorPartialEq; +use orion::numbers::{FixedTrait, FP8x23}; + +#[test] +#[available_gas(2000000000)] +fn test_lrn_fp8x23() { + let input_0 = input_0::input_0(); + let z_0 = output_0::output_0(); + + let y_0 = NNTrait::lrn(@input_0, 3, Option::Some(FP8x23 { mag: 1677, sign: false }),Option::Some(FP8x23 { mag: 4194304, sign: false }),Option::Some(FP8x23 { mag: 16777216, sign: false })); + + assert_eq(y_0, z_0); +} diff --git a/tests/nodes/lrn_fp8x23/input_0.cairo b/tests/nodes/lrn_fp8x23/input_0.cairo new file mode 100644 index 000000000..693e8cd08 --- /dev/null +++ b/tests/nodes/lrn_fp8x23/input_0.cairo @@ -0,0 +1,47 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP8x23Tensor, FP8x23TensorAdd}; +use orion::numbers::{FixedTrait, FP8x23}; + +fn input_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(2); + shape.append(4); + shape.append(2); + shape.append(2); + + let mut data = ArrayTrait::new(); + data.append(FP8x23 { mag: 157361506, sign: false }); + data.append(FP8x23 { mag: 65947556, sign: false }); + data.append(FP8x23 { mag: 184958702, sign: true }); + data.append(FP8x23 { mag: 121472923, sign: false }); + data.append(FP8x23 { mag: 180907185, sign: true }); + data.append(FP8x23 { mag: 2217758, sign: false }); + data.append(FP8x23 { mag: 103839573, sign: true }); + data.append(FP8x23 { mag: 220429352, sign: false }); + data.append(FP8x23 { mag: 229977910, sign: true }); + data.append(FP8x23 { mag: 190142360, sign: false }); + data.append(FP8x23 { mag: 80417469, sign: false }); + data.append(FP8x23 { mag: 233216920, sign: true }); + data.append(FP8x23 { mag: 23249529, sign: false }); + data.append(FP8x23 { mag: 55705405, sign: false }); + data.append(FP8x23 { mag: 249837752, sign: false }); + data.append(FP8x23 { mag: 184155206, sign: true }); + data.append(FP8x23 { mag: 71269904, sign: true }); + data.append(FP8x23 { mag: 109102145, sign: true }); + data.append(FP8x23 { mag: 164635440, sign: true }); + data.append(FP8x23 { mag: 50362409, sign: true }); + data.append(FP8x23 { mag: 213977248, sign: true }); + data.append(FP8x23 { mag: 176365894, sign: true }); + data.append(FP8x23 { mag: 119986317, sign: false }); + data.append(FP8x23 { mag: 57149935, sign: true }); + data.append(FP8x23 { mag: 83110014, sign: true }); + data.append(FP8x23 { mag: 116234081, sign: false }); + data.append(FP8x23 { mag: 173484779, sign: true }); + data.append(FP8x23 { mag: 176794109, sign: true }); + data.append(FP8x23 { mag: 16253546, sign: false }); + data.append(FP8x23 { mag: 147203787, sign: false }); + data.append(FP8x23 { mag: 144635843, sign: true }); + data.append(FP8x23 { mag: 188210025, sign: false }); + TensorTrait::new(shape.span(), data.span()) +} diff --git a/tests/nodes/lrn_fp8x23/output_0.cairo b/tests/nodes/lrn_fp8x23/output_0.cairo new file mode 100644 index 000000000..0414d1863 --- /dev/null +++ b/tests/nodes/lrn_fp8x23/output_0.cairo @@ -0,0 +1,47 @@ +use core::array::{ArrayTrait, SpanTrait}; +use orion::operators::tensor::{TensorTrait, Tensor}; +use orion::operators::tensor::{FP8x23Tensor, FP8x23TensorAdd}; +use orion::numbers::{FixedTrait, FP8x23}; + +fn output_0() -> Tensor { + let mut shape = ArrayTrait::::new(); + shape.append(2); + shape.append(4); + shape.append(2); + shape.append(2); + + let mut data = ArrayTrait::new(); + data.append(FP8x23 { mag: 111256816, sign: false }); + data.append(FP8x23 { mag: 46580759, sign: false }); + data.append(FP8x23 { mag: 128277313, sign: true }); + data.append(FP8x23 { mag: 83611630, sign: false }); + data.append(FP8x23 { mag: 126017564, sign: true }); + data.append(FP8x23 { mag: 1543992, sign: false }); + data.append(FP8x23 { mag: 72682307, sign: true }); + data.append(FP8x23 { mag: 155433087, sign: false }); + data.append(FP8x23 { mag: 157125508, sign: true }); + data.append(FP8x23 { mag: 132534076, sign: false }); + data.append(FP8x23 { mag: 56185980, sign: false }); + data.append(FP8x23 { mag: 163415641, sign: true }); + data.append(FP8x23 { mag: 16276062, sign: false }); + data.append(FP8x23 { mag: 38837883, sign: false }); + data.append(FP8x23 { mag: 172269841, sign: false }); + data.append(FP8x23 { mag: 125158185, sign: true }); + data.append(FP8x23 { mag: 49451868, sign: true }); + data.append(FP8x23 { mag: 75700225, sign: true }); + data.append(FP8x23 { mag: 116359950, sign: true }); + data.append(FP8x23 { mag: 35514562, sign: true }); + data.append(FP8x23 { mag: 146666826, sign: true }); + data.append(FP8x23 { mag: 123872486, sign: true }); + data.append(FP8x23 { mag: 83903259, sign: false }); + data.append(FP8x23 { mag: 39659332, sign: true }); + data.append(FP8x23 { mag: 57700200, sign: true }); + data.append(FP8x23 { mag: 82042811, sign: false }); + data.append(FP8x23 { mag: 121547869, sign: true }); + data.append(FP8x23 { mag: 124950489, sign: true }); + data.append(FP8x23 { mag: 11480852, sign: false }); + data.append(FP8x23 { mag: 102419757, sign: false }); + data.append(FP8x23 { mag: 99907452, sign: true }); + data.append(FP8x23 { mag: 133080503, sign: false }); + TensorTrait::new(shape.span(), data.span()) +}