Merge pull request #1970 from alfred-bratterud/0.13-release

0.13 release

Author: alfreb

.dockerignore

@@ -0,0 +1,3 @@
+build_*
+IncludeOS_install/
+**/build

.gitignore

@@ -17,9 +17,13 @@
 *.img
 *.img.*
 *.gz
+*.tar
+*nacl.txt
 .DS_Store
 nbproject
 dummy.disk
+callgraph.svg
+massif.out.*
 
 !Dockerfile
 
@@ -35,3 +39,15 @@ test/**/*.log
 CMakeFiles*
 CMakeCache*
 cmake_install.cmake
+
+# Name of installation folder
+IncludeOS_install
+
+# Vim
+*.swp
+
+#CLion
+.idea/
+
+# Starbase disk file
+lib/uplink/starbase/disk

CMakeLists.txt

@@ -32,7 +32,7 @@ set(SCRIPTS ${CMAKE_INSTALL_PREFIX}/includeos/scripts)
 set(CPP_VERSION c++17)
 
 # create OS version string from git describe (used in CXX flags)
-execute_process(COMMAND git describe --dirty
+execute_process(COMMAND git describe --tags --dirty
 	        WORKING_DIRECTORY ${INCLUDEOS_ROOT}
 	        OUTPUT_VARIABLE OS_VERSION)
 string(STRIP ${OS_VERSION} OS_VERSION)
@@ -48,28 +48,27 @@ else()
   set(DEFAULT_VM "vm.cpu_feat.json") # vmrunner
 endif(cpu_feat_vanilla)
 
-option(threading "Compile with SMP/threading support" OFF)
+option(smp "Compile with SMP (multiprocessing)" OFF)
 
 option(silent "Disable most output during OS boot" OFF)
 
 option (undefined_san "Enable undefined-behavior sanitizer" OFF)
+option (thin_lto "Enable the Thin LTO plugin" OFF)
+option (full_lto "Enable full LTO (compatibility)" OFF)
 
 # create random hex string as stack protector canary
 string(RANDOM LENGTH 16 ALPHABET 0123456789ABCDEF STACK_PROTECTOR_VALUE)
 
-set(CAPABS "${CAPABS} -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_PROTECTOR_VALUE}")
+set(CAPABS "${CAPABS} -g -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_PROTECTOR_VALUE}")
 
 # Various global defines
 # * NO_DEBUG disables output from the debug macro
 # * OS_TERMINATE_ON_CONTRACT_VIOLATION provides classic assert-like output from Expects / Ensures
-set(CAPABS "${CAPABS} -DNO_DEBUG=1 -DOS_TERMINATE_ON_CONTRACT_VIOLATION ")
-
-set(WARNS "-Wall -Wextra")
+set(CAPABS "${CAPABS} -DNO_DEBUG=1 -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_LIBCPP_HAS_MUSL_LIBC -D_GNU_SOURCE -D__includeos__")
+set(WARNS "-Wall -Wextra") # -Werror
 
 # configure options
-option(debug "Build with debugging symbols (OBS: Dramatically increases binary size)" OFF)
-option(debug-info "Build like \"all\" but with debugging output (i.e. the 'debug'-macro) enabled" OFF)
-option(debug-all "Build with debugging symbols + debugging output, i.e. \"debug\" + \"debug-info\"" OFF)
+option(debug "Build with no optimizations" OFF)
 option(minimal "Build for minimal size" OFF)
 option(stripped "reduce size" OFF)
 
@@ -96,15 +95,6 @@ if(minimal)
 	set(OPTIMIZE "-Os")
 endif(minimal)
 
-# Set debug options
-if(debug OR debug-all)
-	set(CAPABS "${CAPABS} -ggdb3 -DGSL_THROW_ON_CONTRACT_VIOLATION")
-endif(debug OR debug-all)
-
-if(debug-info OR debug-all)
-	set(CAPABS "${CAPABS} -UNO_DEBUG")
-endif(debug-info OR debug-all)
-
 if(silent)
 	set(CAPABS "${CAPABS} -DNO-INFO=1")
 endif(silent)
@@ -116,6 +106,11 @@ set(CAPABS "${CAPABS} ${OPTIMIZE}")
 if (undefined_san)
   set(CAPABS "${CAPABS} -fsanitize=undefined -fno-sanitize=vptr")
 endif()
+if (thin_lto)
+  set(CAPABS "${CAPABS} -flto=thin")
+elseif(full_lto)
+  set(CAPABS "${CAPABS} -flto=full")
+endif()
 
 # object format needs to be set BEFORE enabling ASM
 # see: https://cmake.org/Bug/bug_relationship_graph.php?bug_id=13166
@@ -131,21 +126,15 @@ endif()
 
 enable_language(ASM_NASM)
 
-
-if (NOT threading)
-  add_definitions(-D_LIBCPP_HAS_NO_THREADS)
-endif()
-
-
 # initialize C and C++ compiler flags
 if (CMAKE_COMPILER_IS_GNUCC)
   # gcc/g++ settings
-  set(CMAKE_CXX_FLAGS "-std=${CPP_VERSION} -MMD ${CAPABS} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c ${LIBCPP_THREADING} -DOS_VERSION=\\\"${OS_VERSION}\\\"")
-  set(CMAKE_C_FLAGS " -MMD ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c -DOS_VERSION=\"\"${OS_VERSION}\"\"")
+  set(CMAKE_CXX_FLAGS "${CAPABS} -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c")
+  set(CMAKE_C_FLAGS " ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c")
 else()
   # these kinda work with llvm
-  set(CMAKE_CXX_FLAGS "-std=${CPP_VERSION} -MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c  ${LIBCPP_THREADING} -DOS_VERSION=\\\"${OS_VERSION}\\\"")
-  set(CMAKE_C_FLAGS "-MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c -DOS_VERSION=\"\"${OS_VERSION}\"\"")
+  set(CMAKE_CXX_FLAGS "${CAPABS} -std=${CPP_VERSION} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c")
+  set(CMAKE_C_FLAGS "${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c")
 endif()
 
 # either download or cross-compile needed libraries
@@ -179,7 +168,9 @@ if(vmbuild)
     SOURCE_DIR ${INCLUDEOS_ROOT}/vmbuild # Where is project located
     BINARY_DIR ${INCLUDEOS_ROOT}/vmbuild/build
     INSTALL_DIR ${BIN} # Where to install
-    CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> # Pass installation folder
+    CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
+     -DINCLUDE_PATH=${CMAKE_INSTALL_PREFIX} # Pass installation folder
+    DEPENDS PrecompiledLibraries
     )
 endif(vmbuild)
 
@@ -201,8 +192,14 @@ if(examples)
 endif(examples)
 
 option(tests "Build unit tests in /test and install lest test framework" OFF)
-if(tests)
+option(lest "Install lest unittest headers" OFF)
+
+if (lest OR tests)
   init_submodule(test/lest)
+  install(DIRECTORY test/lest/include/lest DESTINATION ${CMAKE_INSTALL_PREFIX}/includeos/include)
+endif()
+
+if(tests)
   enable_testing()
   ExternalProject_Add(unittests
     PREFIX unittests
@@ -272,8 +269,13 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/elf-toolchain.cmake DESTINATION
 # Install seed
 install(DIRECTORY seed/ DESTINATION includeos/seed)
 
-# Install boot util
+# Install executable scripts
 install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/boot DESTINATION bin)
+install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-run DESTINATION bin)
+install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-callgraph DESTINATION bin)
+install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-debug DESTINATION bin)
+install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-gprof DESTINATION bin)
+install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-pgo DESTINATION bin)
 
 # Install scripts
 install(PROGRAMS

CONTRIBUTE.md

@@ -2,6 +2,6 @@
 
 You're very welcome to [clone, edit and send pull-request](https://help.github.com/articles/using-pull-requests). 
 
-**Please read the contribution gudelines on the wiki: [Conributing to IncludeOS](https://github.com/hioa-cs/IncludeOS/wiki/Contributing-to-IncludeOS)**
+**Please read the contribution guidelines: [Contributing to IncludeOS](http://includeos.readthedocs.io/en/latest/Contributing-to-IncludeOS.html)**
 
 ## Thank you!

Dockerfile

@@ -1,31 +1,106 @@
-FROM ubuntu:xenial
+FROM ubuntu:xenial as base
+
+RUN apt-get update && apt-get -y install \
+    sudo \
+    curl \
+    locales \
+    && rm -rf /var/lib/apt/lists/*
+RUN locale-gen en_US.UTF-8
+ENV LANG=en_US.UTF-8 \
+    LANGUAGE=en_US:en \
+    LC_ALL=en_US.UTF-8
+
+# Add fixuid to change permissions for bind-mounts. Set uid to same as host with -u <uid>:<guid>
+RUN addgroup --gid 1000 docker && \
+    adduser --uid 1000 --ingroup docker --home /home/docker --shell /bin/sh --disabled-password --gecos "" docker && \
+    usermod -aG sudo docker && \
+    sed -i.bkp -e \
+      's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \
+      /etc/sudoers
+RUN USER=docker && \
+    GROUP=docker && \
+    curl -SsL https://github.com/boxboat/fixuid/releases/download/v0.3/fixuid-0.3-linux-amd64.tar.gz | tar -C /usr/local/bin -xzf - && \
+    chown root:root /usr/local/bin/fixuid && \
+    chmod 4755 /usr/local/bin/fixuid && \
+    mkdir -p /etc/fixuid && \
+    printf "user: $USER\ngroup: $GROUP\npaths:\n  - /service\n" > /etc/fixuid/config.yml
+
+ARG CUSTOM_TAG
+LABEL org.label-schema.schema-version="1.0" \
+      org.label-schema.name="IncludeOS builder" \
+      org.label-schema.vendor="IncludeOS" \
+      org.label-schema.version=$CUSTOM_TAG \
+      org.label-schema.vcs-url="https://github.com/hioa-cs/includeos"
+
+RUN echo "LANG=C.UTF-8" > /etc/default/locale
+
+#########################
+FROM base as source-build
 
 RUN apt-get update && apt-get -y install \
     git \
+    lsb-release \
     net-tools \
-    sudo \
     wget \
-&& rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/*
 
-RUN useradd --create-home -s /bin/bash ubuntu
-RUN adduser ubuntu sudo
-RUN echo -n 'ubuntu:ubuntu' | chpasswd
+RUN mkdir -p /root/IncludeOS
+WORKDIR /root/IncludeOS
+COPY . .
 
-# Enable passwordless sudo for users under the "sudo" group
-RUN sed -i.bkp -e \
-      's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \
-      /etc/sudoers
+# Ability to specify custom tag that overwrites any existing tag. This will then match what IncludeOS reports itself.
+ARG CUSTOM_TAG
+RUN git describe --tags --dirty > /ios_version.txt
+RUN : ${CUSTOM_TAG:=$(git describe --tags)} && git tag -d $(git describe --tags); git tag $CUSTOM_TAG && git describe --tags --dirty > /custom_tag.txt
+
+# Installation
+RUN ./install.sh -n
+
+#############################
+FROM base as grubify
+
+RUN apt-get update && apt-get -y install \
+  dosfstools \
+  grub-pc
 
-USER ubuntu
+COPY --from=source-build /usr/local/includeos/scripts/grubify.sh /home/ubuntu/IncludeOS_install/includeos/scripts/grubify.sh
 
-ADD . /home/ubuntu/IncludeOS
-WORKDIR /home/ubuntu/IncludeOS
+ENTRYPOINT ["fixuid", "/home/ubuntu/IncludeOS_install/includeos/scripts/grubify.sh"]
 
-RUN sudo apt-get update && \
-    sudo do_bridge="" ./etc/install_all_source.sh \
-&& sudo rm -rf /var/lib/apt/lists/*
+###########################
+FROM base as build
+
+RUN apt-get update && apt-get -y install \
+    git \
+    clang-5.0 \
+    cmake \
+    nasm \
+    python-pip \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip install pystache antlr4-python2-runtime && \
+    apt-get remove -y python-pip && \
+    apt autoremove -y
+
+ARG VCS_REF="Check file /ios_version.txt inside container"
+LABEL org.label-schema.description="Build a service using IncludeOS" \
+      org.label-schema.vcs-ref=$VCS_REF \
+      org.label-schema.docker.cmd="docker run -v $PWD:/service <container>"
 
-VOLUME /service
 WORKDIR /service
 
-CMD ./run.sh
+COPY --from=source-build  /usr/local/includeos /usr/local/includeos/
+COPY --from=source-build  /usr/local/bin/boot /usr/local/bin/boot
+COPY --from=source-build  /root/IncludeOS/etc/install_dependencies_linux.sh /
+COPY --from=source-build  /root/IncludeOS/etc/use_clang_version.sh /
+COPY --from=source-build  /root/IncludeOS/lib/uplink/starbase /root/IncludeOS/lib/uplink/starbase/
+COPY --from=source-build  /ios_version.txt /
+COPY --from=source-build  /custom_tag.txt /
+COPY --from=source-build  /root/IncludeOS/etc/docker_entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
+
+CMD mkdir -p build && \
+  cd build && \
+  cp $(find /usr/local/includeos -name chainloader) /service/build/chainloader && \
+  echo "IncludeOS version:" $(cat /ios_version.txt) "tag:" $(cat /custom_tag.txt) && \
+  cmake .. && \
+  make

NaCl

@@ -27,12 +27,12 @@ IncludeOS is free software, with "no warranties or restrictions of any kind".
 * **Extreme memory footprint**: A minimal bootable 64-bit web server, including operating system components and a anything needed from the C/C++ standard libraries is currently 2.5 MB.
 * **KVM, VirtualBox and VMWare support** with full virtualization, using [x86 hardware virtualization](https://en.wikipedia.org/wiki/X86_virtualization), available on most modern x86 CPUs. IncludeOS will run on any x86 hardware platform, even on a physical x86 computer, given appropriate drivers. Officially, we develop for- and test on [Linux KVM](http://www.linux-kvm.org/page/Main_Page), and VMWare [ESXi](https://www.vmware.com/products/esxi-and-esx.html)/[Fusion](https://www.vmware.com/products/fusion.html) which means that you can run your IncludeOS service on Linux, Microsoft Windows and macOS, as well as on cloud providers such as [Google Compute Engine](http://www.includeos.org/blog/2017/includeos-on-google-compute-engine.html), [OpenStack](https://www.openstack.org/) and VMWare [vcloud](https://www.vmware.com/products/vcloud-suite.html).
 * **Instant boot:** IncludeOS on Qemu/kvm boots in about 300ms but IBM Research has also integrated IncludeOS with [Solo5/uKVM](https://github.com/Solo5/solo5), providing boot times as low as 10 milliseconds.
-* **C++11/14 support**
-    * Full C++11/14 language support with [clang](http://clang.llvm.org) v3.8 and later.
+* **Modern C++ support**
+    * Full C++11/14/17 language support with [clang](http://clang.llvm.org) 5 and later.
     * Standard C++ library (STL) [libc++](http://libcxx.llvm.org) from [LLVM](http://llvm.org/).
     * Exceptions and stack unwinding (currently using [libgcc](https://gcc.gnu.org/onlinedocs/gccint/Libgcc.html)).
     * *Note:* Certain language features, such as threads and filestreams are currently missing backend support.
-* **Standard C library** using [newlib](https://sourceware.org/newlib/) from [Red Hat](http://www.redhat.com/).
+* **Standard C library** using [musl libc](http://www.musl-libc.org/).
 * **Virtio and vmxnet3 Network drivers** with DMA. [Virtio](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=virtio) provides a highly efficient and widely supported I/O virtualization. vmxnet3 is the VMWare equivalent.
 * **A highly modular TCP/IP-stack**.
 
@@ -42,11 +42,11 @@ A longer list of features and limitations can be found on our [documentation sit
 
 ### Set custom location and compiler
 
-By default the project is installed to /usr/local/includeos.
+By default the project is installed to `/usr/local/includeos`.
 
-However, it is recommended to choose a custom location as well as select the compiler we want clang to find. In this document we assume you install IncludeOS in your home directory, in the folder ~/includeos.
+However, it is recommended to choose a custom location as well as select the compiler we want clang to find. In this document we assume you install IncludeOS in your home directory, in the folder `~/includeos`.
 
-To do this we can edit ~/.bash_profile (mac os) or ~/.bashrc (linux), adding these lines at the end of the file:
+To do this we can edit `~/.bash_profile` (mac os) or `~/.bashrc` (linux), adding these lines at the end of the file:
 
 ```
     export INCLUDEOS_PREFIX=~/includeos/

README.md

@@ -1,4 +1,4 @@
-// -*-C++-*-
+// -*-C++-*-
 // This file is a part of the IncludeOS unikernel - www.includeos.org
 //
 // Copyright 2017 Oslo and Akershus University College of Applied Sciences
@@ -31,19 +31,24 @@ extern void __arch_poweroff();
 extern void __arch_reboot();
 extern void __arch_enable_legacy_irq(uint8_t);
 extern void __arch_disable_legacy_irq(uint8_t);
+extern void __arch_system_deactivate();
 
 extern void __arch_install_irq(uint8_t, void(*)());
 extern void __arch_subscribe_irq(uint8_t);
 extern void __arch_unsubscribe_irq(uint8_t);
 extern void __arch_preempt_forever(void(*)());
-
+extern inline void __arch_read_memory_barrier() noexcept;
+extern inline void __arch_write_memory_barrier() noexcept;
 inline void __arch_hw_barrier() noexcept;
 inline void __sw_barrier() noexcept;
 
 extern uint64_t __arch_system_time() noexcept;
 extern timespec __arch_wall_clock() noexcept;
 inline uint64_t __arch_cpu_cycles() noexcept;
 
+inline void __arch_hw_barrier() noexcept {
+  __sync_synchronize();
+}
 
 inline void __sw_barrier() noexcept
 {

api/arch.hpp

@@ -19,7 +19,7 @@
 #ifndef i686_ARCH_HPP
 #define i686_ARCH_HPP
 
-#include <arch/x86.hpp>
+#define ARCH_x86
 
 inline uint64_t __arch_cpu_cycles() noexcept {
   uint64_t ret;
@@ -28,4 +28,6 @@ inline uint64_t __arch_cpu_cycles() noexcept {
 }
 
 
+constexpr uintptr_t __arch_max_canonical_addr = 0xffffffff;
+
 #endif