From 68fe0cddf5bc8da1f0276f69a119cec8012bf418 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 11:39:52 -0700 Subject: [PATCH 01/16] Travis CI to test that website links are valid --- .travis.yml | 29 +++++++++++++++++++++-------- website/README.md | 12 ++++++++++++ website/linkchecker.sh | 4 ++-- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index ad47d1b2a8a..29d6372f93f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,18 +24,31 @@ env: before_install: # download and install libunwind - - wget -q http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz - - tar xfz libunwind-1.1.tar.gz - - cd libunwind-1.1 && ./configure --prefix=/usr && sudo make --silent install && cd .. + #- wget -q http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz + #- tar xfz libunwind-1.1.tar.gz + #- cd libunwind-1.1 && ./configure --prefix=/usr && sudo make --silent install && cd .. # download and install bazel - - wget -q 'https://github.com/bazelbuild/bazel/releases/download/0.1.2/bazel-0.1.2-installer-linux-x86_64.sh' - - chmod +x bazel-0.1.2-installer-linux-x86_64.sh - - ./bazel-0.1.2-installer-linux-x86_64.sh --user + #- wget -q 'https://github.com/bazelbuild/bazel/releases/download/0.1.2/bazel-0.1.2-installer-linux-x86_64.sh' + #- chmod +x bazel-0.1.2-installer-linux-x86_64.sh + #- ./bazel-0.1.2-installer-linux-x86_64.sh --user + + - wget -q https://github.com/spf13/hugo/releases/download/v0.15/hugo_0.15_amd64.deb + - sudo dpkg -i hugo*.deb + - hugo version + - pip install --user requests==2.9.0 + - pip install --user LinkChecker + - linkchecker --version + +before_script: + - (cd website && npm install && npm install gulp && which gulp) script: - which gcc-4.8 - gcc --version - which g++-4.8 - - g++ --version - - scripts/travis/build.sh && scripts/travis/test.sh + - g++ --version + - which gulp + - (cd website && make site) + - echo "Foo" + #- scripts/travis/build.sh && scripts/travis/test.sh diff --git a/website/README.md b/website/README.md index 88b62d70ba4..eebfe0bcae5 100644 --- a/website/README.md +++ b/website/README.md @@ -74,6 +74,18 @@ browser from the command line: $ open http://localhost:1313/heron ``` +## Checking Links + +To verify that the links in the docs are all valid, run `make linkchecker`, which will produce a +report of broken links. If `linkchecker` fails to install or run properly, you can install it manually. +Note that `linkchecker` versions 9.2 and 9.3 require the python `requests` >= 2.2.0 and < 2.10.0. + +```bash +$ pip uninstall requests +$ pip install requests==2.9.0 +$ pip install linkchecker +``` + ## Publishing the Site The content on the [twitter.github.io/heron](http://twitter.github.io/heron) website is what is diff --git a/website/linkchecker.sh b/website/linkchecker.sh index e621ed95d1c..66720deebcc 100755 --- a/website/linkchecker.sh +++ b/website/linkchecker.sh @@ -1,7 +1,7 @@ -#! bin/bash +#!/bin/bash #linkchecker removes previous runs. -pip install linkchecker +which linkchecker || pip install linkchecker rm -f linkchecker-errors.csv && rm -f linkchecker-out.csv linkchecker public/index.html --no-warnings -F csv #uses error code: on fail, write linkchecker-errors.csv for debugging From e17b7b8dd88e0e420cd1c1c7973148005f644f48 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 12:04:30 -0700 Subject: [PATCH 02/16] chaining commands together --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 29d6372f93f..7bbe3cec4db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,6 +49,4 @@ script: - which g++-4.8 - g++ --version - which gulp - - (cd website && make site) - - echo "Foo" - #- scripts/travis/build.sh && scripts/travis/test.sh + - (cd website && make site || echo 'FAILED: build of website docs failed.') && scripts/travis/build.sh && scripts/travis/test.sh From dddca51c736e8686385ce9fd7b47befd62acf778 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 12:16:52 -0700 Subject: [PATCH 03/16] chaining commands together --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7bbe3cec4db..90b9cfaf031 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,4 +49,4 @@ script: - which g++-4.8 - g++ --version - which gulp - - (cd website && make site || echo 'FAILED: build of website docs failed.') && scripts/travis/build.sh && scripts/travis/test.sh + - (cd website && make site || echo 'FAILED: build of website docs failed.') && cd - && scripts/travis/build.sh && scripts/travis/test.sh From 7345bfb661a9188e937aaf0dcc45caf94972909f Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 12:44:45 -0700 Subject: [PATCH 04/16] chaining commands together --- .travis.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 90b9cfaf031..6b4450eb51a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,15 +24,16 @@ env: before_install: # download and install libunwind - #- wget -q http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz - #- tar xfz libunwind-1.1.tar.gz - #- cd libunwind-1.1 && ./configure --prefix=/usr && sudo make --silent install && cd .. + - wget -q http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz + - tar xfz libunwind-1.1.tar.gz + - cd libunwind-1.1 && ./configure --prefix=/usr && sudo make --silent install && cd .. # download and install bazel - #- wget -q 'https://github.com/bazelbuild/bazel/releases/download/0.1.2/bazel-0.1.2-installer-linux-x86_64.sh' - #- chmod +x bazel-0.1.2-installer-linux-x86_64.sh - #- ./bazel-0.1.2-installer-linux-x86_64.sh --user + - wget -q 'https://github.com/bazelbuild/bazel/releases/download/0.1.2/bazel-0.1.2-installer-linux-x86_64.sh' + - chmod +x bazel-0.1.2-installer-linux-x86_64.sh + - ./bazel-0.1.2-installer-linux-x86_64.sh --user + # required for building the website docs - wget -q https://github.com/spf13/hugo/releases/download/v0.15/hugo_0.15_amd64.deb - sudo dpkg -i hugo*.deb - hugo version @@ -49,4 +50,5 @@ script: - which g++-4.8 - g++ --version - which gulp - - (cd website && make site || echo 'FAILED: build of website docs failed.') && cd - && scripts/travis/build.sh && scripts/travis/test.sh + - (cd website && make site || echo 'FAILED: build of website docs failed.') + - scripts/travis/build.sh && scripts/travis/test.sh From b9b46ba7424e271c7f00775799aa2641ce3d04a0 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 12:50:13 -0700 Subject: [PATCH 05/16] chaining commands together --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b4450eb51a..76703746b74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,6 +49,5 @@ script: - gcc --version - which g++-4.8 - g++ --version - - which gulp - - (cd website && make site || echo 'FAILED: build of website docs failed.') + - (cd website && make site) || echo 'FAILED: build of website docs failed.' - scripts/travis/build.sh && scripts/travis/test.sh From a4fc0bdd7fc632da485fe95363ca4a87e18d3bc2 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 13:01:51 -0700 Subject: [PATCH 06/16] escape : for yml --- .travis.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 76703746b74..ca2baf82fee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,14 +24,14 @@ env: before_install: # download and install libunwind - - wget -q http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz - - tar xfz libunwind-1.1.tar.gz - - cd libunwind-1.1 && ./configure --prefix=/usr && sudo make --silent install && cd .. +# - wget -q http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz +# - tar xfz libunwind-1.1.tar.gz +# - cd libunwind-1.1 && ./configure --prefix=/usr && sudo make --silent install && cd .. # download and install bazel - - wget -q 'https://github.com/bazelbuild/bazel/releases/download/0.1.2/bazel-0.1.2-installer-linux-x86_64.sh' - - chmod +x bazel-0.1.2-installer-linux-x86_64.sh - - ./bazel-0.1.2-installer-linux-x86_64.sh --user +# - wget -q 'https://github.com/bazelbuild/bazel/releases/download/0.1.2/bazel-0.1.2-installer-linux-x86_64.sh' +# - chmod +x bazel-0.1.2-installer-linux-x86_64.sh +# - ./bazel-0.1.2-installer-linux-x86_64.sh --user # required for building the website docs - wget -q https://github.com/spf13/hugo/releases/download/v0.15/hugo_0.15_amd64.deb @@ -49,5 +49,5 @@ script: - gcc --version - which g++-4.8 - g++ --version - - (cd website && make site) || echo 'FAILED: build of website docs failed.' - - scripts/travis/build.sh && scripts/travis/test.sh + - (cd website && make site) || echo 'FAILED\: build of website docs failed.' + #- scripts/travis/build.sh && scripts/travis/test.sh From 3668a74e5e9d6a9e30820a061dc381b9135b5153 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 13:09:45 -0700 Subject: [PATCH 07/16] escape : for yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ca2baf82fee..dbc75762691 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,5 +49,5 @@ script: - gcc --version - which g++-4.8 - g++ --version - - (cd website && make site) || echo 'FAILED\: build of website docs failed.' + - "(cd website && make site || echo 'FAILED: build of website docs failed.')" #- scripts/travis/build.sh && scripts/travis/test.sh From 455b652a6fd220590a0cb4a2ab7b8147a03cf4f4 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 13:14:04 -0700 Subject: [PATCH 08/16] remove logging --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dbc75762691..5d1be938243 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,5 +49,5 @@ script: - gcc --version - which g++-4.8 - g++ --version - - "(cd website && make site || echo 'FAILED: build of website docs failed.')" + - (cd website && make site) #- scripts/travis/build.sh && scripts/travis/test.sh From c148956d04af7277ce0be06e130850a379af3019 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 14:02:13 -0700 Subject: [PATCH 09/16] Fixing return codes --- website/Makefile | 1 + website/linkchecker.sh | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/website/Makefile b/website/Makefile index 82454486ccc..bcc3924cda8 100755 --- a/website/Makefile +++ b/website/Makefile @@ -1,6 +1,7 @@ setup: @brew list hugo || brew install hugo @npm install + @which gulp || npm install gulp serve: @hugo server --watch diff --git a/website/linkchecker.sh b/website/linkchecker.sh index 66720deebcc..088d6b751e8 100755 --- a/website/linkchecker.sh +++ b/website/linkchecker.sh @@ -1,14 +1,22 @@ #!/bin/bash +set -e + #linkchecker removes previous runs. which linkchecker || pip install linkchecker rm -f linkchecker-errors.csv && rm -f linkchecker-out.csv -linkchecker public/index.html --no-warnings -F csv + +set +e +linkchecker public/index.html --no-warnings -F csv +STATUS=$? +set -e + #uses error code: on fail, write linkchecker-errors.csv for debugging -if [[ $? == 1 ]]; then +if [[ $STATUS != 0 ]]; then cut -sd ';' -f 1,2 linkchecker-out.csv | tr ';' ' ' | \ awk '{ print $2 " " $1}' | sort -u >> linkchecker-errors.csv; - echo "linkchecker failed - check linkchecker-errors.csv"; + echo "linkchecker failed - check linkchecker-errors.csv"; + exit $STATUS else echo "linkchecker passes"; fi From 86a41bc66bd3f111c415f5ff51beeef83671f64d Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 14:15:58 -0700 Subject: [PATCH 10/16] getting build to fail on site failure --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d1be938243..2d879f05f3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,14 +24,14 @@ env: before_install: # download and install libunwind -# - wget -q http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz -# - tar xfz libunwind-1.1.tar.gz -# - cd libunwind-1.1 && ./configure --prefix=/usr && sudo make --silent install && cd .. + - wget -q http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz + - tar xfz libunwind-1.1.tar.gz + - cd libunwind-1.1 && ./configure --prefix=/usr && sudo make --silent install && cd .. # download and install bazel -# - wget -q 'https://github.com/bazelbuild/bazel/releases/download/0.1.2/bazel-0.1.2-installer-linux-x86_64.sh' -# - chmod +x bazel-0.1.2-installer-linux-x86_64.sh -# - ./bazel-0.1.2-installer-linux-x86_64.sh --user + - wget -q 'https://github.com/bazelbuild/bazel/releases/download/0.1.2/bazel-0.1.2-installer-linux-x86_64.sh' + - chmod +x bazel-0.1.2-installer-linux-x86_64.sh + - ./bazel-0.1.2-installer-linux-x86_64.sh --user # required for building the website docs - wget -q https://github.com/spf13/hugo/releases/download/v0.15/hugo_0.15_amd64.deb @@ -50,4 +50,4 @@ script: - which g++-4.8 - g++ --version - (cd website && make site) - #- scripts/travis/build.sh && scripts/travis/test.sh + - scripts/travis/build.sh && scripts/travis/test.sh From 054cbad81af49fb92cc43dc46e8cda8c61c127f8 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 14:23:45 -0700 Subject: [PATCH 11/16] getting build to fail on site failure --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2d879f05f3c..c8fec301e57 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,5 +49,4 @@ script: - gcc --version - which g++-4.8 - g++ --version - - (cd website && make site) - - scripts/travis/build.sh && scripts/travis/test.sh + - (cd website && make site) && scripts/travis/build.sh && scripts/travis/test.sh From c21c834ee0cc9d2a0d54af10efc3aca76271148b Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 14:34:15 -0700 Subject: [PATCH 12/16] Verify we success with good linkcheck --- website/linkchecker.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/linkchecker.sh b/website/linkchecker.sh index 088d6b751e8..806cf955e7a 100755 --- a/website/linkchecker.sh +++ b/website/linkchecker.sh @@ -7,7 +7,8 @@ which linkchecker || pip install linkchecker rm -f linkchecker-errors.csv && rm -f linkchecker-out.csv set +e -linkchecker public/index.html --no-warnings -F csv +linkchecker public/docs/resources/index.html --no-warnings -F csv +#linkchecker public/index.html --no-warnings -F csv STATUS=$? set -e From 5458b99c914527a67683dff632226c9154096513 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 14:41:36 -0700 Subject: [PATCH 13/16] Revert testing stuff --- .travis.yml | 8 ++++---- website/linkchecker.sh | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index c8fec301e57..765209d7c31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,9 @@ before_install: - chmod +x bazel-0.1.2-installer-linux-x86_64.sh - ./bazel-0.1.2-installer-linux-x86_64.sh --user +before_script: # required for building the website docs + - (cd website && npm install && npm install gulp && which gulp) - wget -q https://github.com/spf13/hugo/releases/download/v0.15/hugo_0.15_amd64.deb - sudo dpkg -i hugo*.deb - hugo version @@ -41,12 +43,10 @@ before_install: - pip install --user LinkChecker - linkchecker --version -before_script: - - (cd website && npm install && npm install gulp && which gulp) - script: - which gcc-4.8 - gcc --version - which g++-4.8 - g++ --version - - (cd website && make site) && scripts/travis/build.sh && scripts/travis/test.sh + # TODO: change below from || to && once all links are fixed so we fail build on bad ones + - (cd website && make site) || scripts/travis/build.sh && scripts/travis/test.sh diff --git a/website/linkchecker.sh b/website/linkchecker.sh index 806cf955e7a..088d6b751e8 100755 --- a/website/linkchecker.sh +++ b/website/linkchecker.sh @@ -7,8 +7,7 @@ which linkchecker || pip install linkchecker rm -f linkchecker-errors.csv && rm -f linkchecker-out.csv set +e -linkchecker public/docs/resources/index.html --no-warnings -F csv -#linkchecker public/index.html --no-warnings -F csv +linkchecker public/index.html --no-warnings -F csv STATUS=$? set -e From af727ad5781538210a5470fda1684a754de4820f Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 16:54:20 -0700 Subject: [PATCH 14/16] Moving pip install into Makefile --- website/Makefile | 1 + website/README.md | 1 + website/linkchecker.sh | 6 +++++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/website/Makefile b/website/Makefile index bcc3924cda8..115091acf56 100755 --- a/website/Makefile +++ b/website/Makefile @@ -2,6 +2,7 @@ setup: @brew list hugo || brew install hugo @npm install @which gulp || npm install gulp + @pip install linkchecker serve: @hugo server --watch diff --git a/website/README.md b/website/README.md index eebfe0bcae5..6839c184fbc 100644 --- a/website/README.md +++ b/website/README.md @@ -13,6 +13,7 @@ Running the Heron documentation locally requires that you have the following ins * [Make](https://www.gnu.org/software/make/) * [Node.js](https://nodejs.org/en/) * [npm](https://www.npmjs.com/) +* [pip](https://pypi.python.org/pypi/pip) ### OS X Setup diff --git a/website/linkchecker.sh b/website/linkchecker.sh index 088d6b751e8..8e611791cd8 100755 --- a/website/linkchecker.sh +++ b/website/linkchecker.sh @@ -2,8 +2,12 @@ set -e +function die { + echo $1 && exit 1 +} + #linkchecker removes previous runs. -which linkchecker || pip install linkchecker +which linkchecker || die 'Linkchecker must be installed to run this script. Exiting' rm -f linkchecker-errors.csv && rm -f linkchecker-out.csv set +e From 5bccabb1cc1b42c3b551d119103f8e3f19445099 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 16:57:42 -0700 Subject: [PATCH 15/16] Moving pip install into Makefile --- website/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/README.md b/website/README.md index 6839c184fbc..57fe2ed9e75 100644 --- a/website/README.md +++ b/website/README.md @@ -79,7 +79,8 @@ $ open http://localhost:1313/heron To verify that the links in the docs are all valid, run `make linkchecker`, which will produce a report of broken links. If `linkchecker` fails to install or run properly, you can install it manually. -Note that `linkchecker` versions 9.2 and 9.3 require the python `requests` >= 2.2.0 and < 2.10.0. +Note that due to this [https://github.com/wummel/linkchecker/pull/657](issue), `linkchecker` +versions 9.2 and 9.3 require the python `requests` >= 2.2.0 and < 2.10.0. ```bash $ pip uninstall requests From b82e0ed84c0f42966d566b64914e55b5e6f87ae4 Mon Sep 17 00:00:00 2001 From: Bill Graham Date: Thu, 19 May 2016 17:25:30 -0700 Subject: [PATCH 16/16] Use semicolon not OR --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 765209d7c31..5fe977de233 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,5 +48,5 @@ script: - gcc --version - which g++-4.8 - g++ --version - # TODO: change below from || to && once all links are fixed so we fail build on bad ones - - (cd website && make site) || scripts/travis/build.sh && scripts/travis/test.sh + # TODO: change below from ; to && once all links are fixed so we fail build on bad ones + - (cd website && make site); scripts/travis/build.sh && scripts/travis/test.sh