specialize undefinedness

.github/workflows/lint.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7.4
+          python-version: 3.x
           architecture: x64
       - name: Checkout PyTorch
         uses: actions/checkout@master
@@ -28,7 +28,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7.4
+          python-version: 3.x
           architecture: x64
       - name: Checkout PyTorch
         uses: actions/checkout@master
@@ -69,7 +69,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7.4
+          python-version: 3.x
           architecture: x64
       - name: Checkout PyTorch
         uses: actions/checkout@master
@@ -84,7 +84,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7.4
+          python-version: 3.x
           architecture: x64
       - name: Checkout PyTorch
         uses: actions/checkout@master
@@ -99,7 +99,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7.4
+          python-version: 3.x
           architecture: x64
       - name: Fetch PyTorch
         uses: actions/checkout@master
@@ -174,7 +174,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7.4
+          python-version: 3.x
           architecture: x64
       - name: Checkout PyTorch
         uses: actions/checkout@master
@@ -209,7 +209,10 @@ jobs:
         run: |
           set -eux
           git remote add upstream https://github.com/pytorch/pytorch
-          git fetch upstream "${{ github.base_ref}}"
+          git fetch upstream "$GITHUB_BASE_REF"
+          BASE_SHA=${{ github.event.pull_request.base.sha }}
+          HEAD_SHA=${{ github.event.pull_request.head.sha }}
+          MERGE_BASE=$(git merge-base $BASE_SHA $HEAD_SHA)
 
           if [[ ! -d build ]]; then
             git submodule update --init --recursive
@@ -238,8 +241,9 @@ jobs:
           # The negative filters below are to exclude files that include onnx_pb.h or
           # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job.
           python tools/clang_tidy.py                  \
+            --verbose                                 \
             --paths torch/csrc/                       \
-            --diff "${{ github.event.pull_request.base.sha}}" \
+            --diff "$MERGE_BASE"                      \
             -g"-torch/csrc/jit/export.cpp"            \
             -g"-torch/csrc/jit/import.cpp"            \
             -g"-torch/csrc/jit/netdef_converter.cpp"  \
@@ -252,6 +256,6 @@ jobs:
           check_name: 'clang-tidy'
           linter_output_path: 'clang-tidy-output.txt'
           commit_sha: ${{ steps.get_pr_tip.outputs.commit_sha }}
-          regex: '^(?<filename>.*?):(?<lineNumber>\d+):(?<columnNumber>\d+): (?<errorDesc>.*?) (?<errorCode>\[.*\])'
+          regex: '^(?<filename>.*?):(?<lineNumber>\d+):(?<columnNumber>\d+): (?<errorDesc>.*?) \[(?<errorCode>.*)\]'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.jenkins/pytorch/test.sh

@@ -110,7 +110,7 @@ test_python_nn() {
 }
 
 test_python_all_except_nn() {
-  time python test/run_test.py --exclude nn --verbose --bring-to-front quantization quantized quantized_tensor quantized_nn_mods quantizer
+  time python test/run_test.py --exclude nn --verbose --bring-to-front quantization quantized quantized_tensor quantized_nn_mods
   assert_git_not_dirty
 }
 

android/build.gradle

@@ -1,33 +1,37 @@
-buildscript {
-    ext {
-        minSdkVersion = 21
-        targetSdkVersion = 28
-        compileSdkVersion = 28
-        buildToolsVersion = '28.0.3'
-
-        coreVersion = "1.2.0"
-        extJUnitVersion = "1.1.1"
-        runnerVersion = "1.2.0"
-        rulesVersion = "1.2.0"
-        junitVersion = "4.12"
+allprojects {
+    if (name == "pytorch_host") {
+        return
     }
 
-    repositories {
-        google()
-        mavenLocal()
-        mavenCentral()
-        jcenter()
-    }
+    buildscript {
+        ext {
+            minSdkVersion = 21
+            targetSdkVersion = 28
+            compileSdkVersion = 28
+            buildToolsVersion = '28.0.3'
+
+            coreVersion = "1.2.0"
+            extJUnitVersion = "1.1.1"
+            runnerVersion = "1.2.0"
+            rulesVersion = "1.2.0"
+            junitVersion = "4.12"
+        }
+
+        repositories {
+            google()
+            mavenLocal()
+            mavenCentral()
+            jcenter()
+        }
 
-    dependencies {
-        classpath 'com.android.tools.build:gradle:3.3.2'
-        classpath "com.jfrog.bintray.gradle:gradle-bintray-plugin:${GRADLE_BINTRAY_PLUGIN_VERSION}"
-        classpath "com.github.dcendents:android-maven-gradle-plugin:${ANDROID_MAVEN_GRADLE_PLUGIN_VERSION}"
-        classpath "org.jfrog.buildinfo:build-info-extractor-gradle:4.9.8"
+        dependencies {
+            classpath 'com.android.tools.build:gradle:3.3.2'
+            classpath "com.jfrog.bintray.gradle:gradle-bintray-plugin:${GRADLE_BINTRAY_PLUGIN_VERSION}"
+            classpath "com.github.dcendents:android-maven-gradle-plugin:${ANDROID_MAVEN_GRADLE_PLUGIN_VERSION}"
+            classpath "org.jfrog.buildinfo:build-info-extractor-gradle:4.9.8"
+        }
     }
-}
 
-allprojects {
     repositories {
         google()
         jcenter()

android/pytorch_android/CMakeLists.txt

@@ -4,7 +4,19 @@ set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_VERBOSE_MAKEFILE ON)
 
 set(pytorch_android_DIR ${CMAKE_CURRENT_LIST_DIR}/src/main/cpp)
-set(libtorch_include_DIR ${pytorch_android_DIR}/libtorch_include/${ANDROID_ABI})
+
+if (ANDROID_ABI)
+  set(libtorch_include_DIR ${pytorch_android_DIR}/libtorch_include/${ANDROID_ABI})
+  set(BUILD_SUBDIR ${ANDROID_ABI})
+else()
+  if (NOT LIBTORCH_HOME)
+    message(FATAL_ERROR
+      "pytorch_android requires LIBTORCH_HOME to be defined for non-Android builds.")
+  endif()
+  set(libtorch_include_DIR ${LIBTORCH_HOME}/include)
+  link_directories(${LIBTORCH_HOME}/lib)
+  set(BUILD_SUBDIR host)
+endif()
 
 message(STATUS "libtorch dir:${libtorch_DIR}")
 
@@ -24,40 +36,58 @@ target_include_directories(pytorch PUBLIC
     ${libtorch_include_DIR}
 )
 
-set(BUILD_DIR ${CMAKE_SOURCE_DIR}/build)
-file(MAKE_DIRECTORY ${BUILD_DIR})
-
 set(fbjni_DIR ${CMAKE_CURRENT_LIST_DIR}/../libs/fbjni/)
-set(fbjni_BUILD_DIR ${BUILD_DIR}/fbjni/${ANDROID_ABI})
+set(fbjni_BUILD_DIR ${CMAKE_BINARY_DIR}/fbjni/${BUILD_SUBDIR})
 
 add_subdirectory(${fbjni_DIR} ${fbjni_BUILD_DIR})
 
-function(import_static_lib name)
-  add_library(${name} STATIC IMPORTED)
-  set_property(
-      TARGET ${name}
-      PROPERTY IMPORTED_LOCATION
-      ${CMAKE_CURRENT_LIST_DIR}/src/main/jniLibs/${ANDROID_ABI}/${name}.a)
-endfunction(import_static_lib)
-
-import_static_lib(libtorch)
-import_static_lib(libc10)
-import_static_lib(libnnpack)
-import_static_lib(libpytorch_qnnpack)
-import_static_lib(libeigen_blas)
-import_static_lib(libcpuinfo)
-import_static_lib(libclog)
-
-target_link_libraries(pytorch
-    fbjni
-    -Wl,--gc-sections
-    -Wl,--whole-archive
-    libtorch
-    -Wl,--no-whole-archive
-    libc10
-    libnnpack
-    libpytorch_qnnpack
-    libeigen_blas
-    libcpuinfo
-    libclog
-)
+if (ANDROID_ABI)
+
+  function(import_static_lib name)
+    add_library(${name} STATIC IMPORTED)
+    set_property(
+        TARGET ${name}
+        PROPERTY IMPORTED_LOCATION
+        ${CMAKE_CURRENT_LIST_DIR}/src/main/jniLibs/${ANDROID_ABI}/${name}.a)
+  endfunction(import_static_lib)
+
+  import_static_lib(libtorch)
+  import_static_lib(libc10)
+  import_static_lib(libnnpack)
+  import_static_lib(libpytorch_qnnpack)
+  import_static_lib(libeigen_blas)
+  import_static_lib(libcpuinfo)
+  import_static_lib(libclog)
+
+    # Link most things statically on Android.
+  target_link_libraries(pytorch
+      fbjni
+      -Wl,--gc-sections
+      -Wl,--whole-archive
+      libtorch
+      -Wl,--no-whole-archive
+      libc10
+      libnnpack
+      libpytorch_qnnpack
+      libeigen_blas
+      libcpuinfo
+      libclog
+  )
+
+else()
+
+  # Prefer dynamic linking on the host
+  target_link_libraries(pytorch
+      fbjni
+      -Wl,--gc-sections
+      -Wl,--whole-archive
+      torch
+      -Wl,--no-whole-archive
+      c10
+      nnpack
+      pytorch_qnnpack
+      cpuinfo
+      clog
+  )
+
+endif()

android/pytorch_android/host/build.gradle

@@ -0,0 +1,33 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// This source code is licensed under the Apache-2 license found in the
+// LICENSE file in the root directory of this source tree.
+
+plugins {
+    id 'java-library'
+}
+
+repositories {
+    mavenLocal()
+    jcenter()
+}
+
+sourceSets {
+    main {
+        java.srcDir '../src/main/java'
+    }
+    test {
+        java {
+            srcDir '../src/androidTest/java'
+            exclude '**/PytorchInstrumented*'
+        }
+        resources.srcDirs = ["../src/androidTest/assets"]
+    }
+}
+
+dependencies {
+    compileOnly 'com.google.code.findbugs:jsr305:3.0.1'
+    implementation 'com.facebook.soloader:nativeloader:0.8.0'
+    implementation 'com.facebook.fbjni:fbjni:0.0.3-SNAPSHOT'
+    testImplementation 'junit:junit:4.12'
+}

android/settings.gradle

@@ -1,4 +1,6 @@
-include ':app', ':pytorch_android', ':fbjni', ':pytorch_android_torchvision'
+include ':app', ':pytorch_android', ':fbjni', ':pytorch_android_torchvision', ':pytorch_host'
 
 project(':fbjni').projectDir = file('libs/fbjni_local')
 project(':pytorch_android_torchvision').projectDir = file('pytorch_android_torchvision')
+
+project(':pytorch_host').projectDir = file('pytorch_android/host')

aten/src/ATen/core/jit_type.h

@@ -556,36 +556,50 @@ struct CAFFE2_API TensorType : public Type {
     return r;
   }
 
+  TensorTypePtr withPossiblyUndefined() {
+    auto r = clone();
+    r->undefined_ = c10::nullopt;
+    return r;
+  }
+
   c10::optional<bool> undefined() const { return undefined_; }
 
   static TensorTypePtr get();
 
   static const TypeKind Kind = TypeKind::TensorType;
 
  private:
-   TensorType(const at::Tensor &tensor)
-       : Type(TypeKind::TensorType), scalar_type_(tensor.scalar_type()),
-         device_(tensor.device()), sizes_(tensor.sizes().size()),
-         strides_(tensor.sizes().size()),
-         requires_grad_(tensor.requires_grad()), undefined_(false) {
-     if (!tensor.is_mkldnn() && !tensor.is_sparse()) {
-       sizes_ = tensor.sizes().vec();
-       strides_ = tensor.strides().vec();
-     }
-        }
-        TensorType(c10::optional<at::ScalarType> scalar_type,
-                   c10::optional<Device> device, const VaryingShape &sizes,
-                   const VaryingStrides &strides,
-                   c10::optional<bool> requires_grad,
-                   c10::optional<bool> undefined = false)
-            : Type(TypeKind::TensorType), scalar_type_(scalar_type),
-              device_(device), sizes_(sizes), strides_(strides),
-              requires_grad_(requires_grad), undefined_(undefined) {}
-
-        TensorTypePtr clone() const {
-          return TensorTypePtr(new TensorType(scalar_type_, device_, sizes_,
-                                              strides_, requires_grad_,
-                                              undefined_));
+  TensorType(const at::Tensor& tensor)
+      : Type(TypeKind::TensorType),
+        scalar_type_(tensor.scalar_type()),
+        device_(tensor.device()),
+        sizes_(tensor.sizes().size()),
+        strides_(tensor.sizes().size()),
+        requires_grad_(tensor.requires_grad()),
+        undefined_(!tensor.defined()) {
+    if (!tensor.is_mkldnn() && !tensor.is_sparse()) {
+      sizes_ = tensor.sizes().vec();
+      strides_ = tensor.strides().vec();
+    }
+  }
+  TensorType(
+      c10::optional<at::ScalarType> scalar_type,
+      c10::optional<Device> device,
+      const VaryingShape& sizes,
+      const VaryingStrides& strides,
+      c10::optional<bool> requires_grad,
+      c10::optional<bool> undefined = false)
+      : Type(TypeKind::TensorType),
+        scalar_type_(scalar_type),
+        device_(device),
+        sizes_(sizes),
+        strides_(strides),
+        requires_grad_(requires_grad),
+        undefined_(undefined) {}
+
+  TensorTypePtr clone() const {
+    return TensorTypePtr(new TensorType(
+        scalar_type_, device_, sizes_, strides_, requires_grad_, undefined_));
   }
 
   static std::vector<int64_t> contiguousStridesOf(at::IntArrayRef sizes) {