From f7adcf299c5209f401b7d7739124732c918b54a7 Mon Sep 17 00:00:00 2001 From: mtar Date: Fri, 25 Feb 2022 13:28:29 +0100 Subject: [PATCH 01/22] Create ci.yaml --- .github/workflows/ci.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..0129f76452 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,36 @@ +name: ci + +on: + pull_request: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + py-version: + - 3.7 + mpi: [ 'mpich', 'openmpi' ] + install-options: [ '.', '.[hdf5,netcdf]' ] + + name: Python ${{ matrix.py-version }} with MPI ${{ matrix.mpi }}; options ${{ matrix.install-options }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup MPI + uses: mpi4py/setup-mpi@v1 + with: + mpi: ${{ matrix.mpi }} + - name: Use Python ${{ matrix.py-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.py-version }} + architecture: x64 + - name: Test + run: | + pip install pytest + pip install ${{ matrix.install-options }} + mpirun -n 4 pytest heat/ From 2ab82b53b44a2d7254142f606363f44905c3e428 Mon Sep 17 00:00:00 2001 From: mtar Date: Fri, 25 Feb 2022 13:44:35 +0100 Subject: [PATCH 02/22] Update ci.yaml --- .github/workflows/ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0129f76452..d77621f77a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,9 @@ name: ci on: + push: + branches: + - enhancement/203-ghactions-matrix pull_request: branches: - master From f261e8e50b764c41fbcc81679e8233affb8618a5 Mon Sep 17 00:00:00 2001 From: mtar Date: Fri, 25 Feb 2022 13:55:08 +0100 Subject: [PATCH 03/22] Update ci.yaml --- .github/workflows/ci.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d77621f77a..0129f76452 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,9 +1,6 @@ name: ci on: - push: - branches: - - enhancement/203-ghactions-matrix pull_request: branches: - master From 9b863a7588f228b836a58f4981dceadd7ec269ab Mon Sep 17 00:00:00 2001 From: mtar Date: Fri, 4 Mar 2022 16:54:17 +0100 Subject: [PATCH 04/22] Create CITATION.cff --- CITATION.cff | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000..8558e36f2c --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,93 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +authors: +- family-names: "Götz" + given-names: "Markus" + orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Debus" + given-names: "Charlotte" + orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Coquelin" + given-names: "Daniel" + orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Krajsek" + given-names: "Kai" + orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Comito" + given-names: "Claudia" + orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Knechtges" + given-names: "Philipp" + orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Hagemeier" + given-names: "Björn" + orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Tarnawa" + given-names: "Michael" + orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Hanselmann" + given-names: "Simon" + orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Siggel" + given-names: "Martin" + orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Basermann" + given-names: "Achim" + orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Streit" + given-names: "Achim" + orcid: "https://orcid.org/0000-0000-0000-0000" +title: "Heat - Helmholtz Analytics Toolkit" +version: 1.1.0 +doi: 10.5281/zenodo.1234 +date-released: 2021-09-21 +url: "https://github.com/helmholtz-analytics/heat" +preferred-citation: + type: conference-paper + authors: + - family-names: "Götz" + given-names: "Markus" + orcid: "https://orcid.org/0000-0000-0000-0000" + - family-names: "Debus" + given-names: "Charlotte" + orcid: "https://orcid.org/0000-0000-0000-0000" + - family-names: "Coquelin" + given-names: "Daniel" + orcid: "https://orcid.org/0000-0000-0000-0000" + - family-names: "Krajsek" + given-names: "Kai" + orcid: "https://orcid.org/0000-0000-0000-0000" + - family-names: "Comito" + given-names: "Claudia" + orcid: "https://orcid.org/0000-0000-0000-0000" + - family-names: "Knechtges" + given-names: "Philipp" + orcid: "https://orcid.org/0000-0000-0000-0000" + - family-names: "Hagemeier" + given-names: "Björn" + orcid: "https://orcid.org/0000-0000-0000-0000" + - family-names: "Tarnawa" + given-names: "Michael" + orcid: "https://orcid.org/0000-0000-0000-0000" + - family-names: "Hanselmann" + given-names: "Simon" + orcid: "https://orcid.org/0000-0000-0000-0000" + - family-names: "Siggel" + given-names: "Martin" + orcid: "https://orcid.org/0000-0000-0000-0000" + - family-names: "Basermann" + given-names: "Achim" + orcid: "https://orcid.org/0000-0000-0000-0000" + - family-names: "Streit" + given-names: "Achim" + orcid: "https://orcid.org/0000-0000-0000-0000" + title: "HeAT -- a Distributed and GPU-accelerated Tensor Framework for Data Analytics}" + year: 2020 + collection-title: "2020 IEEE International Conference on Big Data (IEEE Big Data 2020)" + collection-doi: 10.1109/BigData50022.2020.9378050 + conference: + name: 2020 IEEE International Conference on Big Data (IEEE Big Data 2020) + date-start: 2020-12-10 + date-end: 2020-12-13 + start: 276 + end: 287 From 2b2622a2220767f4f0a9de00703fb027f5fb8acc Mon Sep 17 00:00:00 2001 From: mtar Date: Mon, 7 Mar 2022 10:14:19 +0100 Subject: [PATCH 05/22] Update CITATION.cff --- CITATION.cff | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 8558e36f2c..7c6eb77d92 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -39,7 +39,6 @@ authors: orcid: "https://orcid.org/0000-0000-0000-0000" title: "Heat - Helmholtz Analytics Toolkit" version: 1.1.0 -doi: 10.5281/zenodo.1234 date-released: 2021-09-21 url: "https://github.com/helmholtz-analytics/heat" preferred-citation: @@ -81,7 +80,7 @@ preferred-citation: - family-names: "Streit" given-names: "Achim" orcid: "https://orcid.org/0000-0000-0000-0000" - title: "HeAT -- a Distributed and GPU-accelerated Tensor Framework for Data Analytics}" + title: "HeAT -- a Distributed and GPU-accelerated Tensor Framework for Data Analytics" year: 2020 collection-title: "2020 IEEE International Conference on Big Data (IEEE Big Data 2020)" collection-doi: 10.1109/BigData50022.2020.9378050 From a15b299beec56be98b19d70a14bdfcf6384ed0b8 Mon Sep 17 00:00:00 2001 From: mtar Date: Tue, 8 Mar 2022 11:01:38 +0100 Subject: [PATCH 06/22] Update ci.yaml different python and pytorch versions --- .github/workflows/ci.yaml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0129f76452..de2b9f1e09 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,22 +1,28 @@ name: ci on: - pull_request: - branches: - - master + pull_request_review: + types: [submitted] jobs: - build: + approved: + if: github.event.review.state == 'approved' runs-on: ubuntu-latest strategy: fail-fast: false matrix: py-version: - 3.7 - mpi: [ 'mpich', 'openmpi' ] + - 3.8 + mpi: [ 'openmpi' ] install-options: [ '.', '.[hdf5,netcdf]' ] + pytorch-version: + - torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio==0.7.2 + - torch==1.8.1+cpu torchvision==0.9.1+cpu torchaudio==0.8.1 + - torch==1.9.0+cpu torchvision==0.10.0+cpu torchaudio==0.9.0 - name: Python ${{ matrix.py-version }} with MPI ${{ matrix.mpi }}; options ${{ matrix.install-options }} + + name: Python ${{ matrix.py-version }} with ${{ matrix.pytorch-version }}; options ${{ matrix.install-options }} steps: - name: Checkout uses: actions/checkout@v2 @@ -32,5 +38,7 @@ jobs: - name: Test run: | pip install pytest + pip install ${{ matrix.pytorch-version }} -f https://download.pytorch.org/whl/torch_stable.html pip install ${{ matrix.install-options }} + mpirun -n 3 pytest heat/ mpirun -n 4 pytest heat/ From 8910bf7252cfd758ad07689ba2d8dfebef519fbb Mon Sep 17 00:00:00 2001 From: mtar Date: Wed, 9 Mar 2022 11:01:06 +0100 Subject: [PATCH 07/22] Update ci.yaml From f8dc8b8232d081520dd3406ab96430ab48aa5ebe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Mar 2022 10:01:46 +0000 Subject: [PATCH 08/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index de2b9f1e09..652fe27580 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,7 @@ jobs: mpi: [ 'openmpi' ] install-options: [ '.', '.[hdf5,netcdf]' ] pytorch-version: - - torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio==0.7.2 + - torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio==0.7.2 - torch==1.8.1+cpu torchvision==0.9.1+cpu torchaudio==0.8.1 - torch==1.9.0+cpu torchvision==0.10.0+cpu torchaudio==0.9.0 From 767eabc4da09ba9a32b5431885277e4dcbce524f Mon Sep 17 00:00:00 2001 From: mtar Date: Wed, 9 Mar 2022 11:07:51 +0100 Subject: [PATCH 09/22] Delete pre-commit.yml --- .github/workflows/pre-commit.yml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index b52d4afe5c..0000000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: pre-commit - -on: - pull_request: - push: - branches: [main] - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.0 From 61cef7f0265d971a4ba64f2f96d07a2bb1b266d8 Mon Sep 17 00:00:00 2001 From: mtar Date: Wed, 9 Mar 2022 11:33:50 +0100 Subject: [PATCH 10/22] Update ci.yaml --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 652fe27580..33237d4424 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,9 +17,9 @@ jobs: mpi: [ 'openmpi' ] install-options: [ '.', '.[hdf5,netcdf]' ] pytorch-version: - - torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio==0.7.2 - - torch==1.8.1+cpu torchvision==0.9.1+cpu torchaudio==0.8.1 - - torch==1.9.0+cpu torchvision==0.10.0+cpu torchaudio==0.9.0 + - 'torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio==0.7.2' + - 'torch==1.8.1+cpu torchvision==0.9.1+cpu torchaudio==0.8.1' + - 'torch==1.9.0+cpu torchvision==0.10.0+cpu torchaudio==0.9.0' name: Python ${{ matrix.py-version }} with ${{ matrix.pytorch-version }}; options ${{ matrix.install-options }} From 74b1a30a955a357702081f9d6d860e2a8d0de63c Mon Sep 17 00:00:00 2001 From: mtar Date: Fri, 11 Mar 2022 17:00:28 +0100 Subject: [PATCH 11/22] Update CITATION.cff --- CITATION.cff | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 7c6eb77d92..b655ef2fcc 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -3,40 +3,28 @@ message: "If you use this software, please cite it as below." authors: - family-names: "Götz" given-names: "Markus" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Debus" given-names: "Charlotte" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Coquelin" given-names: "Daniel" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Krajsek" given-names: "Kai" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Comito" given-names: "Claudia" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Knechtges" given-names: "Philipp" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Hagemeier" given-names: "Björn" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Tarnawa" given-names: "Michael" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Hanselmann" given-names: "Simon" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Siggel" given-names: "Martin" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Basermann" given-names: "Achim" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Streit" given-names: "Achim" - orcid: "https://orcid.org/0000-0000-0000-0000" title: "Heat - Helmholtz Analytics Toolkit" version: 1.1.0 date-released: 2021-09-21 @@ -46,40 +34,28 @@ preferred-citation: authors: - family-names: "Götz" given-names: "Markus" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Debus" given-names: "Charlotte" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Coquelin" given-names: "Daniel" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Krajsek" given-names: "Kai" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Comito" given-names: "Claudia" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Knechtges" given-names: "Philipp" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Hagemeier" given-names: "Björn" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Tarnawa" given-names: "Michael" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Hanselmann" given-names: "Simon" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Siggel" given-names: "Martin" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Basermann" given-names: "Achim" - orcid: "https://orcid.org/0000-0000-0000-0000" - family-names: "Streit" given-names: "Achim" - orcid: "https://orcid.org/0000-0000-0000-0000" title: "HeAT -- a Distributed and GPU-accelerated Tensor Framework for Data Analytics" year: 2020 collection-title: "2020 IEEE International Conference on Big Data (IEEE Big Data 2020)" From 93cd8319b6193946ee7e443686370e0cc7d5a229 Mon Sep 17 00:00:00 2001 From: mtar Date: Fri, 11 Mar 2022 17:17:39 +0100 Subject: [PATCH 12/22] Update tutorial.ipynb delete example with different split axis --- scripts/tutorial.ipynb | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/scripts/tutorial.ipynb b/scripts/tutorial.ipynb index f2ce191bd2..95cc6e3465 100644 --- a/scripts/tutorial.ipynb +++ b/scripts/tutorial.ipynb @@ -1044,38 +1044,6 @@ "a + b" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The example below will show that it is also possible to use operations on tensors with different split and the proper result calculated. However, this should be used seldomly and with small data amounts only, as it entails sending large amounts of data over the network." - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0/2) tensor([[9., 9., 9., 9., 9., 9.],\n", - "(0/2) [9., 9., 9., 9., 9., 9.]])\n", - "(1/2) tensor([[9., 9., 9., 9., 9., 9.],\n", - "(1/2) [9., 9., 9., 9., 9., 9.]])" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = ht.full((4, 6,), 8, split=0)\n", - "b = ht.ones((4, 6,), split=1)\n", - "a + b" - ] - }, { "cell_type": "markdown", "metadata": {}, From dd1b83d6d8b36cb4a70eefc631f00277b0745fee Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Tue, 15 Mar 2022 10:01:41 +0100 Subject: [PATCH 13/22] Delete logo_heAT.pdf Removal of old logo --- doc/images/logo_heAT.pdf | Bin 1690 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 doc/images/logo_heAT.pdf diff --git a/doc/images/logo_heAT.pdf b/doc/images/logo_heAT.pdf deleted file mode 100644 index d839eade2b7bed5a6a288bde60136fc4475df68d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1690 zcmZuy2T&AO7!E`ST@jdAG6=>;If0yTyT?@|6z{m>EJ#r-5M{kv?oRQR?CnXPU=WO& z!4e%xtdwA*h!Sv=&I#BDL$FLx(I^(ipeQ1!d3%%)CvSH4?fc&Q`}_W71xsb_mK8T*C2z(kGYdZreHe?2fYW-#(e{i7?L__g%DjFqM7mk;IDIQ19S`l|Xo6(`aQ)pq&u zM3jX#e3q@Q^T)2Ah~ALt>f*^m_0Q7>1Mk#u<8sCG?oIj3`zV=eXMepn?y|g&o~_!A zyj)!^s?a}$R*h4V_OI^fG?TA7GY6O*Y52n$i}Nsa<260cGRt?B+9lBQt_*b-ZhL{#KLcOWA!Wl;ATzjn>5#q{Ao+K_V-(6 z{hBt>JqVl5zIsDCA6LCSm}G5X8NKE2d6m^KtDx1~%CN!INw**S-16-{?{F=Xm+WMW zdsyR zX>I(j$hzfp=lN;xCO_%1@0wyPykB8RZiy`g*x>B#=Un1? zn@9U8f-84L$|x@PV(M#K&R%6sSI*g=1uuLK+49*>tLm1W>9;xarSwS0&7d-=)8cXs zd;5^_aF28S92e|tpg@!7_<2{}s(mg(Oq$u=+$YnQBv1>*9bXuCwq?t<0rOK6X05nq zmVRuO!|zjB^~Mj`L6*v+>A^|fVVwP?w;o`J+T)i;RDAV~Rm6&&gA#Vx1;^8^wPp^j zQGVvr+@tm!(@co(JL>A-Y&Ey(r03iYo9gbF7j3u$>(;(-30y=*&2BXolqpoUtC}ut zUw}KV^Nov`C_G-Du{!hmtGf#MZQr(!dCA985JIOK;R*a=7{<_wnIW-+Vj;87tUSl~ zXJTAHJS`8SA=`krHv_=I!BMyX9Em@`07r?#H{>APfN%JW=;4m(0i0zCu>{}*<7xu1 z0A|t~j8lY;hN7d?UP7p}_yRH>L_i1yA|d9%5IfGrVT&eo)ax+l2ZihOv5aM9!YHf&G-V)0R}y$iN^H_9iBS0h1{{uz z6H4ew1EnPNfXPqjxHy>zM*G#jaq1aa&LXW!5947{5jy6(feCw@0>L;1!4#=7C}D?l zRpMHT1egcL_rOr#s-fvvFAhvLMAZ}?tI;a;9weo9b2Ax|!2U;TNu87_l&jQ>i*iwv z$MwYE;ELHO*9Ar0#@Q5(vpMXv1gG&BQf<=46iPo*ntux#PZ7}wEDVB<4Itq2J^3IS zj9?fSgO`dKKsSn^Vi8Q)rx*s)_6dgmkE}=}{`4Hi6~X)QUs=pk46`_j(G;$YAt=)y uButw~K$n?fgpwo;n81f`j6xSp0w(vSV(Mv}qD>KEJe~+)u>zLLkbeM} Date: Wed, 23 Mar 2022 20:50:48 +0530 Subject: [PATCH 14/22] ht.nonzero() returns tuple of 1-D arrays instead of n-D arrays --- heat/core/indexing.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/heat/core/indexing.py b/heat/core/indexing.py index ac7598b9b9..3a85931f53 100644 --- a/heat/core/indexing.py +++ b/heat/core/indexing.py @@ -53,14 +53,14 @@ def nonzero(x: DNDarray) -> DNDarray: """ sanitation.sanitize_in(x) + lcl_nonzero = torch.transpose(torch.nonzero(input=x.larray, as_tuple=False), 0, 1) + if x.split is None: # if there is no split then just return the values from torch - lcl_nonzero = torch.nonzero(input=x.larray, as_tuple=False) gout = list(lcl_nonzero.size()) is_split = None else: # a is split - lcl_nonzero = torch.nonzero(input=x.larray, as_tuple=False) _, _, slices = x.comm.chunk(x.shape, x.split) lcl_nonzero[..., x.split] += slices[x.split].start gout = list(lcl_nonzero.size()) @@ -69,18 +69,17 @@ def nonzero(x: DNDarray) -> DNDarray: if x.ndim == 1: lcl_nonzero = lcl_nonzero.squeeze(dim=1) - for g in range(len(gout) - 1, -1, -1): - if gout[g] == 1: - del gout[g] - - return DNDarray( - lcl_nonzero, - gshape=tuple(gout), - dtype=types.canonical_heat_type(lcl_nonzero.dtype), - split=is_split, - device=x.device, - comm=x.comm, - balanced=False, + + return tuple( + DNDarray( + lcl_nonzero, + gshape=tuple(gout), + dtype=types.canonical_heat_type(lcl_nonzero.dtype), + split=is_split, + device=x.device, + comm=x.comm, + balanced=False, + ) ) From aeb5b6e1c55ff841bcb42971d761ef3674151035 Mon Sep 17 00:00:00 2001 From: Ashwath V A Date: Thu, 24 Mar 2022 20:31:39 +0530 Subject: [PATCH 15/22] Updated documentation and Unit-tests --- heat/core/indexing.py | 16 ++++++++-------- heat/core/tests/test_indexing.py | 15 ++++++++------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/heat/core/indexing.py b/heat/core/indexing.py index 3a85931f53..d7d0520847 100644 --- a/heat/core/indexing.py +++ b/heat/core/indexing.py @@ -13,13 +13,13 @@ __all__ = ["nonzero", "where"] -def nonzero(x: DNDarray) -> DNDarray: +def nonzero(x: DNDarray) -> Tuple: """ - Return a :class:`~heat.core.dndarray.DNDarray` containing the indices of the elements that are non-zero.. (using ``torch.nonzero``) + Return a Tuple of :class:`~heat.core.dndarray.DNDarray`s, one for each dimension of a, + containing the indices of the non-zero elements in that dimension. (using ``torch.nonzero``) If ``x`` is split then the result is split in the 0th dimension. However, this :class:`~heat.core.dndarray.DNDarray` can be UNBALANCED as it contains the indices of the non-zero elements on each node. - Returns an array with one entry for each dimension of ``x``, containing the indices of the non-zero elements in that dimension. - The values in ``x`` are always tested and returned in row-major, C-style order. + The values in ``x`` are always tested and returned in column-major, F-style order. The corresponding non-zero values can be obtained with: ``x[nonzero(x)]``. Parameters @@ -32,10 +32,8 @@ def nonzero(x: DNDarray) -> DNDarray: >>> import heat as ht >>> x = ht.array([[3, 0, 0], [0, 4, 1], [0, 6, 0]], split=0) >>> ht.nonzero(x) - DNDarray([[0, 0], - [1, 1], - [1, 2], - [2, 1]], dtype=ht.int64, device=cpu:0, split=0) + (DNDarray([0, 1, 1, 2], dtype=ht.int64, device=cpu:0, split=None), + DNDarray([0, 1, 2, 1], dtype=ht.int64, device=cpu:0, split=None)) >>> y = ht.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], split=0) >>> y > 3 DNDarray([[False, False, False], @@ -48,6 +46,8 @@ def nonzero(x: DNDarray) -> DNDarray: [2, 0], [2, 1], [2, 2]], dtype=ht.int64, device=cpu:0, split=0) + (DNDarray([1, 1, 1, 2, 2, 2], dtype=ht.int64, device=cpu:0, split=None), + DNDarray([0, 1, 2, 0, 1, 2], dtype=ht.int64, device=cpu:0, split=None)) >>> y[ht.nonzero(y > 3)] DNDarray([4, 5, 6, 7, 8, 9], dtype=ht.int64, device=cpu:0, split=0) """ diff --git a/heat/core/tests/test_indexing.py b/heat/core/tests/test_indexing.py index 4707aa28ab..6a38141d73 100644 --- a/heat/core/tests/test_indexing.py +++ b/heat/core/tests/test_indexing.py @@ -1,4 +1,5 @@ import heat as ht +from torch import transpose from .test_suites.basic_test import TestCase @@ -9,18 +10,18 @@ def test_nonzero(self): a = ht.array([[1, 2, 3], [4, 5, 2], [7, 8, 9]], split=None) cond = a > 3 nz = ht.nonzero(cond) - self.assertEqual(nz.gshape, (5, 2)) - self.assertEqual(nz.dtype, ht.int64) - self.assertEqual(nz.split, None) + self.assertEqual(len(nz), 2) + self.assertEqual(len(nz[0]), 5) + self.assertEqual(nz[0].dtype, ht.int64) # split a = ht.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], split=1) cond = a > 3 nz = cond.nonzero() - self.assertEqual(nz.gshape, (6, 2)) - self.assertEqual(nz.dtype, ht.int64) - self.assertEqual(nz.split, 0) - a[nz] = 10.0 + self.assertEqual(len(nz), 2) + self.assertEqual(len(nz[0]), 6) + self.assertEqual(nz[0].dtype, ht.int64) + a[nz] = 10 self.assertEqual(ht.all(a[nz] == 10), 1) def test_where(self): From 420f064dbd681e6584bf2836fcafa7231c1ebb62 Mon Sep 17 00:00:00 2001 From: Claudia Comito <39374113+ClaudiaComito@users.noreply.github.com> Date: Fri, 25 Mar 2022 09:52:17 +0100 Subject: [PATCH 16/22] replace x.larray with local_x --- heat/core/indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heat/core/indexing.py b/heat/core/indexing.py index f49cb4d708..109b401f64 100644 --- a/heat/core/indexing.py +++ b/heat/core/indexing.py @@ -56,7 +56,7 @@ def nonzero(x: DNDarray) -> Tuple: except AttributeError: raise TypeError("Input must be a DNDarray, is {}".format(type(x))) - lcl_nonzero = torch.transpose(torch.nonzero(input=x.larray, as_tuple=False), 0, 1) + lcl_nonzero = torch.transpose(torch.nonzero(input=local_x, as_tuple=False), 0, 1) if x.split is None: # if there is no split then just return the values from torch From a00ed6105a0473966064abb1212d9500778fd3de Mon Sep 17 00:00:00 2001 From: Ashwath V A Date: Mon, 28 Mar 2022 22:15:49 +0530 Subject: [PATCH 17/22] Code fixes --- heat/core/dndarray.py | 4 ++-- heat/core/indexing.py | 5 ++--- heat/core/tests/test_indexing.py | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/heat/core/dndarray.py b/heat/core/dndarray.py index 5ffce7a777..6034d446a7 100644 --- a/heat/core/dndarray.py +++ b/heat/core/dndarray.py @@ -872,7 +872,7 @@ def __getitem__(self, key: Union[int, Tuple[int, ...], List[int, ...]]) -> DNDar output_split = None # data are not distributed or split dimension is not affected by indexing - if not self.is_distributed or key[self.split] == slice(None): + if not self.is_distributed() or key[self.split] == slice(None): return DNDarray( self.larray[key], gshape=output_shape, @@ -1654,7 +1654,7 @@ def __set(arr: DNDarray, value: DNDarray): raise Exception("Advanced indexing is not supported yet") split = self.split - if not self.is_distributed or key[split] == slice(None): + if not self.is_distributed() or key[split] == slice(None): return __set(self[key], value) if isinstance(key[split], slice): diff --git a/heat/core/indexing.py b/heat/core/indexing.py index 109b401f64..ddca09700d 100644 --- a/heat/core/indexing.py +++ b/heat/core/indexing.py @@ -56,7 +56,7 @@ def nonzero(x: DNDarray) -> Tuple: except AttributeError: raise TypeError("Input must be a DNDarray, is {}".format(type(x))) - lcl_nonzero = torch.transpose(torch.nonzero(input=local_x, as_tuple=False), 0, 1) + lcl_nonzero = torch.nonzero(input=local_x, as_tuple=False) if x.split is None: # if there is no split then just return the values from torch @@ -73,8 +73,7 @@ def nonzero(x: DNDarray) -> Tuple: gout[0] = x.comm.allreduce(gout[0], MPI.SUM) is_split = 0 - if x.ndim == 1: - lcl_nonzero = lcl_nonzero.squeeze(dim=1) + lcl_nonzero = lcl_nonzero.transpose(0, 1) return tuple( DNDarray( diff --git a/heat/core/tests/test_indexing.py b/heat/core/tests/test_indexing.py index 6a38141d73..58c7410456 100644 --- a/heat/core/tests/test_indexing.py +++ b/heat/core/tests/test_indexing.py @@ -1,5 +1,4 @@ import heat as ht -from torch import transpose from .test_suites.basic_test import TestCase From d4a8813b96e4e0814043e6045fb8b61ed285d776 Mon Sep 17 00:00:00 2001 From: Ashwath V A Date: Wed, 30 Mar 2022 20:01:15 +0530 Subject: [PATCH 18/22] Fix return type of nonzero function and gout value --- heat/core/indexing.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/heat/core/indexing.py b/heat/core/indexing.py index ddca09700d..90db6b81cd 100644 --- a/heat/core/indexing.py +++ b/heat/core/indexing.py @@ -13,7 +13,7 @@ __all__ = ["nonzero", "where"] -def nonzero(x: DNDarray) -> Tuple: +def nonzero(x: DNDarray) -> Tuple[DNDarray, ...]: """ Return a Tuple of :class:`~heat.core.dndarray.DNDarray`s, one for each dimension of a, containing the indices of the non-zero elements in that dimension. (using ``torch.nonzero``) @@ -59,7 +59,9 @@ def nonzero(x: DNDarray) -> Tuple: lcl_nonzero = torch.nonzero(input=local_x, as_tuple=False) if x.split is None: - # if there is no split then just return the values from torch + # if there is no split then just return the transpose of values from torch + lcl_nonzero = lcl_nonzero.transpose(0, 1) + gout = list(lcl_nonzero.size()) is_split = None else: @@ -68,13 +70,14 @@ def nonzero(x: DNDarray) -> Tuple: _, displs = x.counts_displs() lcl_nonzero[..., x.split] += displs[x.comm.rank] del displs + + lcl_nonzero = lcl_nonzero.transpose(0, 1) + # get global size of split dimension gout = list(lcl_nonzero.size()) gout[0] = x.comm.allreduce(gout[0], MPI.SUM) is_split = 0 - lcl_nonzero = lcl_nonzero.transpose(0, 1) - return tuple( DNDarray( lcl_nonzero, From 67fcdc88f5c980c27832f18ec9420337b29300bc Mon Sep 17 00:00:00 2001 From: Ashwath V A Date: Wed, 30 Mar 2022 20:09:43 +0530 Subject: [PATCH 19/22] Made sure DNDarray meta-data is available to the tuple members --- heat/core/indexing.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/heat/core/indexing.py b/heat/core/indexing.py index 90db6b81cd..384474ee7b 100644 --- a/heat/core/indexing.py +++ b/heat/core/indexing.py @@ -78,18 +78,23 @@ def nonzero(x: DNDarray) -> Tuple[DNDarray, ...]: gout[0] = x.comm.allreduce(gout[0], MPI.SUM) is_split = 0 - return tuple( - DNDarray( - lcl_nonzero, - gshape=tuple(gout), - dtype=types.canonical_heat_type(lcl_nonzero.dtype), - split=is_split, - device=x.device, - comm=x.comm, - balanced=False, - ) + non_zero_indices = list( + [ + DNDarray( + dim_indices, + gshape=tuple(dim_indices.size()), + dtype=types.canonical_heat_type(lcl_nonzero.dtype), + split=is_split, + device=x.device, + comm=x.comm, + balanced=False, + ) + for dim_indices in lcl_nonzero + ] ) + return tuple(non_zero_indices) + DNDarray.nonzero = lambda self: nonzero(self) DNDarray.nonzero.__doc__ = nonzero.__doc__ From 39103fa303059cd9c43a4c6d974d3a456a8b5ba4 Mon Sep 17 00:00:00 2001 From: Ashwath V A Date: Fri, 1 Apr 2022 19:44:11 +0530 Subject: [PATCH 20/22] Transpose before if-branching + adjustments to accomodate it --- heat/core/indexing.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/heat/core/indexing.py b/heat/core/indexing.py index 384474ee7b..0aaa40da2a 100644 --- a/heat/core/indexing.py +++ b/heat/core/indexing.py @@ -15,11 +15,11 @@ def nonzero(x: DNDarray) -> Tuple[DNDarray, ...]: """ - Return a Tuple of :class:`~heat.core.dndarray.DNDarray`s, one for each dimension of a, - containing the indices of the non-zero elements in that dimension. (using ``torch.nonzero``) - If ``x`` is split then the result is split in the 0th dimension. However, this :class:`~heat.core.dndarray.DNDarray` + Return a Tuple of :class:`~heat.core.dndarray.DNDarray`s, one for each dimension of ``x``, + containing the indices of the non-zero elements in that dimension. If ``x`` is split then + the result is split in the 0th dimension. However, this :class:`~heat.core.dndarray.DNDarray` can be UNBALANCED as it contains the indices of the non-zero elements on each node. - The values in ``x`` are always tested and returned in column-major, F-style order. + The values in ``x`` are always tested and returned in row-major, C-style order. The corresponding non-zero values can be obtained with: ``x[nonzero(x)]``. Parameters @@ -56,11 +56,10 @@ def nonzero(x: DNDarray) -> Tuple[DNDarray, ...]: except AttributeError: raise TypeError("Input must be a DNDarray, is {}".format(type(x))) - lcl_nonzero = torch.nonzero(input=local_x, as_tuple=False) + lcl_nonzero = torch.nonzero(input=local_x, as_tuple=False).transpose(0, 1) if x.split is None: # if there is no split then just return the transpose of values from torch - lcl_nonzero = lcl_nonzero.transpose(0, 1) gout = list(lcl_nonzero.size()) is_split = None @@ -68,14 +67,12 @@ def nonzero(x: DNDarray) -> Tuple[DNDarray, ...]: # a is split # adjust local indices along split dimension _, displs = x.counts_displs() - lcl_nonzero[..., x.split] += displs[x.comm.rank] + lcl_nonzero[x.split] += displs[x.comm.rank] del displs - lcl_nonzero = lcl_nonzero.transpose(0, 1) - # get global size of split dimension gout = list(lcl_nonzero.size()) - gout[0] = x.comm.allreduce(gout[0], MPI.SUM) + gout[1] = x.comm.allreduce(gout[1], MPI.SUM) is_split = 0 non_zero_indices = list( From 3ed205cd8b117382ad401b48d63f0e25ceb76f7d Mon Sep 17 00:00:00 2001 From: Ashwath V A Date: Tue, 5 Apr 2022 09:18:07 +0530 Subject: [PATCH 21/22] Fixed global shape assignment --- heat/core/indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heat/core/indexing.py b/heat/core/indexing.py index 0aaa40da2a..0452000c2f 100644 --- a/heat/core/indexing.py +++ b/heat/core/indexing.py @@ -79,7 +79,7 @@ def nonzero(x: DNDarray) -> Tuple[DNDarray, ...]: [ DNDarray( dim_indices, - gshape=tuple(dim_indices.size()), + gshape=tuple(gout), dtype=types.canonical_heat_type(lcl_nonzero.dtype), split=is_split, device=x.device, From 70dded62a57c0e2761b902811ee6758d3bc1ee79 Mon Sep 17 00:00:00 2001 From: Ashwath V A Date: Fri, 8 Apr 2022 09:40:52 +0530 Subject: [PATCH 22/22] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddb06676e4..3dca59403b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - [#876](https://github.com/helmholtz-analytics/heat/pull/876) Make examples work (Lasso and kNN) - [#894](https://github.com/helmholtz-analytics/heat/pull/894) Change inclusion of license file - [#884](https://github.com/helmholtz-analytics/heat/pull/884) Added capabilities for PyTorch 1.10.0, this is now the recommended version to use. +- [#937](https://github.com/helmholtz-analytics/heat/pull/937) Modified `ht.nonzero()` to return a tuple of 1-D arrays containing the non-zero indices in each dimension. ## Bug Fixes - [#826](https://github.com/helmholtz-analytics/heat/pull/826) Fixed `__setitem__` handling of distributed `DNDarray` values which have a different shape in the split dimension