Address feedback 2 cassandra

buildenv/jenkins/common/build.groovy

@@ -113,17 +113,17 @@ def checkout_pullrequest() {
     // Cannot depend on a change from the same repo as the source repo
     def SOURCE_REPO_MESSAGE
     switch (params.ghprbGhRepository) {
-        case "eclipse/openj9-omr":
+        case ~/.*\/openj9-omr/:
             omr_bool = true
             OMR_PR = params.ghprbPullId
             SOURCE_REPO_MESSAGE = "PullRequest source repo appears to be OpenJ9-OMR"
             break
-        case "eclipse/openj9":
+        case ~/.*\/openj9/:
             openj9_bool = true
             OPENJ9_PR = params.ghprbPullId
             SOURCE_REPO_MESSAGE = "PullRequest source repo appears to be OpenJ9"
             break
-        case ~/ibmruntimes(.*)/:
+        case ~/.*openj9-openjdk-jdk.*/:
             openjdk_bool = true
             OPENJDK_PR = params.ghprbPullId
             SOURCE_REPO_MESSAGE = "PullRequest source repo appears to be OpenJDK"
@@ -198,7 +198,7 @@ def checkout_pullrequest() {
         checkout_pullrequest(OPENJDK_PR, "ibmruntimes/openj9-openjdk-jdk${JDK_REPO_SUFFIX}")
     }
 
-    sh "bash ./get_source.sh ${EXTRA_GETSOURCE_OPTIONS}"
+    sh "bash ./get_source.sh ${EXTRA_GETSOURCE_OPTIONS} ${OPENJ9_REPO_OPTION} ${OPENJ9_BRANCH_OPTION} ${OPENJ9_SHA_OPTION} ${OMR_REPO_OPTION} ${OMR_BRANCH_OPTION} ${OMR_SHA_OPTION}"
 
     // Checkout dependent PRs, if any were specified
     if (openj9_bool) {
@@ -225,6 +225,8 @@ def checkout_pullrequest(ID, REPO) {
                 fetchRef("pull/${ID}/merge", "pr/${ID}/merge" )
                 checkoutRef ("pr/${ID}/merge")
             } catch (e) {
+                // If merge ref doesn't exist, checkout PR base branch.
+                // Not going to fix this to work for internal PRs.
                 def JSONFILE = ""
                 def BASEREF = ""
                 def URL = "https://api.github.com/repos/${REPO}/pulls/${ID}"

buildenv/jenkins/common/pipeline-functions.groovy

@@ -714,14 +714,19 @@ def setup_pull_request_single_comment(parsedComment) {
     // Setup JDK VERSIONS
     switch (ghprbGhRepository) {
         case ~/.*openj9-openjdk-jdk.*/:
+            // <org>/openj9-openjdk-jdk<version>(-zos)?
             def tmp_version = ghprbGhRepository.substring(ghprbGhRepository.indexOf('-jdk')+4)
             if ("${tmp_version}" == "") {
                 tmp_version = 'next'
             }
+            if (tmp_version.contains('-')) {
+                // Strip off '-zos'
+                tmp_version = tmp_version.substring(0, tmp_version.indexOf('-'))
+            }
             RELEASES.add(tmp_version)
             minCommentSize = 3
             break
-        case "eclipse/openj9":
+        case ~/.*\/openj9(-omr)?/:
             def tmpVersions = parsedComment[3+offset].tokenize(',')
             tmpVersions.each { version ->
                 echo "VERSION:'${version}'"

buildenv/jenkins/jobs/pipelines/Pipeline-Build-Test-All.groovy

@@ -135,7 +135,8 @@ SHORT_NAMES = ['all' : ['ppc64le_linux','ppc64le_linux_xl','s390x_linux','s390x_
             'osxxl' : ['x86-64_mac_xl'],
             'alinux64' : ['aarch64_linux'],
             'alinux64xl' : ['aarch64_linux_xl'],
-            'alinux64largeheap' : ['aarch64_linux_xl']]
+            'alinux64largeheap' : ['aarch64_linux_xl'],
+            'zos' : ['s390x_zos']]
 
 // Initialize all PARAMETERS (params) to Groovy Variables even if they are not passed
 echo "Initialize all PARAMETERS..."
@@ -191,14 +192,14 @@ echo "SCM_REPO:'${SCM_REPO}'"
 SCM_BRANCH = 'refs/heads/master'
 if (params.SCM_BRANCH) {
     SCM_BRANCH = params.SCM_BRANCH
-} else if (ghprbPullId && ghprbGhRepository == 'eclipse/openj9') {
+} else if (ghprbPullId && ghprbGhRepository ==~ /.*\/openj9/) {
     SCM_BRANCH = sha1
 }
 echo "SCM_BRANCH:'${SCM_BRANCH}'"
 SCM_REFSPEC = ''
 if (params.SCM_REFSPEC) {
     SCM_REFSPEC = params.SCM_REFSPEC
-} else if (ghprbPullId && ghprbGhRepository == 'eclipse/openj9') {
+} else if (ghprbPullId && ghprbGhRepository ==~ /.*\/openj9/) {
     SCM_REFSPEC = "+refs/pull/${ghprbPullId}/merge:refs/remotes/origin/pr/${ghprbPullId}/merge"
 }
 echo "SCM_REFSPEC:'${SCM_REFSPEC}'"

buildenv/jenkins/variables/defaults.yml

@@ -183,8 +183,7 @@ ppc64le_linux:
   node_labels:
     build: 'ci.role.build && hw.arch.ppc64le && sw.os.ubuntu'
   extra_configure_options:
-    8: '--enable-jitserver'
-    11: '--enable-jitserver'
+    all: '--enable-jitserver'
   build_env:
     vars: 'PATH+GCC7=/usr/local/gcc-7.5.0/bin LD_LIBRARY_PATH=/usr/local/gcc-7.5.0/lib64:$LD_LIBRARY_PATH'
 #========================================#
@@ -199,10 +198,8 @@ ppc64le_linux_xl:
   extends: ['ppc64le_linux', 'largeheap']
   excluded_tests:
     8:
-      - sanity.openjdk
       - special.system
     11:
-      - sanity.openjdk
       - special.system
 #========================================#
 # Linux PPCLE 64bits Compressed Pointers /w JITSERVER
@@ -237,10 +234,7 @@ s390x_linux:
 s390x_linux_xl:
   extends: ['s390x_linux', 'largeheap']
   excluded_tests:
-    8:
-      - sanity.openjdk
-    11:
-      - sanity.openjdk
+     11:
       - special.system
 #========================================#
 # Linux S390 64bits Compressed Pointers /w JITSERVER
@@ -253,10 +247,10 @@ s390x_linux_jit:
 ppc64_aix:
   extends: ['debuginfo', 'freemarker', 'openjdk_reference_repo', 'openssl']
   boot_jdk:
-    8: '/usr/java7'
-    11: '/usr/java11_64'
-    14: '/usr/java13_64'
-    next: '/usr/java13_64'
+    8: '/opt/java7'
+    11: '/opt/java11_64'
+    14: '/opt/java13_64'
+    next: '/opt/java13_64'
   release:
     all: 'aix-ppc64-server-release'
     8: 'aix-ppc64-normal-server-release'
@@ -276,10 +270,6 @@ ppc64_aix:
       14: 'PATH+XLC=/opt/IBM/xlC/16.1.0/bin:/opt/IBM/xlc/16.1.0/bin CC=xlclang CXX=xlclang++'
       next: 'PATH+XLC=/opt/IBM/xlC/16.1.0/bin:/opt/IBM/xlc/16.1.0/bin CC=xlclang CXX=xlclang++'
   excluded_tests:
-    8:
-      - sanity.openjdk
-    11:
-      - sanity.openjdk
       - special.system
 #========================================#
 # Linux x86 64bits Compressed Pointers
@@ -298,8 +288,7 @@ x86-64_linux:
   node_labels:
     build: 'ci.role.build && hw.arch.x86 && sw.os.cent.6'
   extra_configure_options:
-    8: '--enable-jitserver'
-    11: '--enable-jitserver'
+    all: '--enable-jitserver'
   build_env:
     cmd: 'source /home/jenkins/set_gcc7.5.0_env'
     vars: 'OPENJ9_JAVA_OPTIONS=-Xdump:system+java:events=systhrow,filter=java/lang/ClassCastException,request=exclusive+prepwalk+preempt'
@@ -323,10 +312,7 @@ x86-64_linux_xl:
   extends: ['x86-64_linux', 'largeheap']
   excluded_tests:
     8:
-      - sanity.openjdk
       - special.system
-    11:
-      - sanity.openjdk
 #========================================#
 # Linux x86 64bits Compressed Pointers / Valhalla
 #========================================#
@@ -375,10 +361,7 @@ x86-64_windows:
   build_env:
     vars: 'PATH+TOOLS=/cygdrive/c/openjdk/LLVM64/bin:/cygdrive/c/openjdk/nasm-2.13.03'
   excluded_tests:
-    8:
-      - sanity.openjdk
     11:
-      - sanity.openjdk
       - special.system
 #========================================#
 # Windows x86 64bits Large Heap
@@ -405,10 +388,7 @@ x86-32_windows:
       8: 'PATH+TOOLS=/cygdrive/c/openjdk/LLVM32/bin:/cygdrive/c/openjdk/nasm-2.13.03'
   excluded_tests:
     8:
-      - sanity.openjdk
       - special.system
-    11:
-      - sanity.openjdk
 #========================================#
 # OSX x86 64bits Compressed Pointers
 #========================================#
@@ -436,10 +416,7 @@ x86-64_mac:
       all: 'OPENJ9_JAVA_OPTIONS=-Xdump:system+java:events=systhrow,filter=java/lang/ClassCastException,request=exclusive+prepwalk+preempt'
       8: 'MACOSX_DEPLOYMENT_TARGET=10.9.0 SDKPATH=/Users/jenkins/Xcode4/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.8.sdk'
   excluded_tests:
-    8:
-      - sanity.openjdk
     11:
-      - sanity.openjdk
       - special.system
 #========================================#
 # OSX x86 64bits Large Heap

debugtools/DDR_VM/src/com/ibm/j9ddr/vm29/j9/stackwalker/JITRegMap.java

@@ -429,7 +429,10 @@ public class JITRegMap {
 					0x11,   /* jit_r17 */
 					0x12,   /* jit_r18 */
 					0x13,   /* jit_r19 */
-					0x14    /* jit_r20 */
+					0x14,   /* jit_r20 */
+					0x1D,   /* jit_r29 */
+					0x1E,   /* jit_r30 */
+					0x1F    /* jit_r31 */
 			};
 
 			jitCalleeSavedRegisterList = new int[] {

debugtools/DDR_VM/src/com/ibm/j9ddr/vm29/j9/stackwalker/JITStackWalker.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2009, 2019 IBM Corp. and others
+ * Copyright (c) 2009, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -293,10 +293,6 @@ public FrameCallbackResult jitWalkStackFrames(WalkState walkState)
 			walkState.arg0EA = walkState.i2jState.a0();
 			returnSP = walkState.i2jState.returnSP();
 			walkState.previousFrameFlags = new UDATA(0);
-			if (returnSP.anyBitsIn(J9_STACK_FLAGS_ARGS_ALIGNED)) {
-				swPrintf(walkState, 2, "I2J args were copied for alignment");
-				walkState.previousFrameFlags = new UDATA(J9_STACK_FLAGS_JIT_ARGS_ALIGNED);
-			}
 			walkState.walkSP = returnSP.untag(3L);
 			swPrintf(walkState, 2, "I2J values: PC = {0}, A0 = {1}, walkSP = {2}, literals = {3}, JIT PC = {4}, pcAddress = {5}, decomp = {6}", 
 					walkState.pc.getHexAddress(),

doc/compiler/memory/MemoryManager.md

@@ -1,5 +1,5 @@
 <!--
-Copyright (c) 2017, 2018 IBM Corp. and others
+Copyright (c) 2017, 2020 IBM Corp. and others
 
 This program and the accompanying materials are made available under
 the terms of the Eclipse Public License 2.0 which accompanies this
@@ -224,3 +224,20 @@ calling it `TR::PersistentSegmentProvider`, and have it use
 `TR::PersistentAllocator`. The `TR::Region` would then be provided this 
 `TR::PersistentSegmentProvider` object.
 
+The mechanism for limiting the memory usage during compilation can be summarized as:
+
+Default size of system segments allocatd by `J9::SystemSegmentProvider` is 16 MB. 
+These system segments are then carved into smaller segments based on the size requested
+which is rounded up to 64 KB. The cumulative memory usage of these system segments is limited 
+by the value of the option `scratchSpaceLimit`. When the scratch space limit is not a multiple
+of system segment size, it is possible that the total memory allocated using system segments
+would go beyond the scratch space limit. However, the logic in `J9::SystemSegmentProvider::request`
+would ensures that the compiler does not touch memory beyond the scratch space limit.
+So the physical memory usage due to scratch space stays with the limit specified. 
+If a request is made that would cause physical memory usage to go beyond the limit,
+then `J9::SystemSegmentProvider::request` would throw `std::bad_alloc`.
+
+When allocating a new system segment `J9::SegmentAllocator::allocate` also checks that the system
+has enough free physical memory. This is done by verifying that the free physical memory is more
+than the system segment size (=16 MB) plus an additional buffer, which is governed by the 
+option `safeReservePhysicalMemoryValue`. If this condition fails, then the compilation thread is suspended.

jcl/src/java.base/share/classes/com/ibm/oti/reflect/AnnotationParser.java

@@ -2,7 +2,7 @@
 package com.ibm.oti.reflect;
 
 /*******************************************************************************
- * Copyright (c) 2010, 2017 IBM Corp. and others
+ * Copyright (c) 2010, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -25,6 +25,9 @@
 
 import java.lang.annotation.Annotation;
 import java.nio.ByteBuffer;
+import java.lang.reflect.Method;
+import java.lang.reflect.Field;
+import java.lang.reflect.Constructor;
 /*[IF Sidecar19-SE]
 import jdk.internal.reflect.ConstantPool;
 /*[ELSE]*/
@@ -33,23 +36,23 @@
 
 public class AnnotationParser {
 
-	public static Annotation[] parseAnnotations(java.lang.reflect.Field field) {
+	public static Annotation[] parseAnnotations(Field field) {
 		return parseAnnotations(getAnnotationsData(field), field.getDeclaringClass());
 	}
 
-	public static Annotation[] parseAnnotations(java.lang.reflect.Constructor constructor) {
+	public static Annotation[] parseAnnotations(Constructor constructor) {
 		return parseAnnotations(getAnnotationsData(constructor), constructor.getDeclaringClass());
 	}
 
-	public static Annotation[] parseAnnotations(java.lang.reflect.Method method) {
+	public static Annotation[] parseAnnotations(Method method) {
 		return parseAnnotations(getAnnotationsData(method), method.getDeclaringClass());
 	}
 
-	public static Annotation[][] parseParameterAnnotations(java.lang.reflect.Constructor constructor) {
+	public static Annotation[][] parseParameterAnnotations(Constructor constructor) {
 		return parseParameterAnnotations(getParameterAnnotationsData(constructor), constructor.getDeclaringClass(), constructor.getParameterTypes().length);
 	}
 
-	public static Annotation[][] parseParameterAnnotations(java.lang.reflect.Method method) {
+	public static Annotation[][] parseParameterAnnotations(Method method) {
 		return parseParameterAnnotations(getParameterAnnotationsData(method), method.getDeclaringClass(), method.getParameterTypes().length);
 	}
 
@@ -65,7 +68,7 @@ public static  byte[] getAnnotationsData(java.lang.Class clazz) {
 		return result;
 	};
 
-	public static Object parseDefaultValue(java.lang.reflect.Method method) {
+	public static Object parseDefaultValue(Method method) {
 		byte[] elementValueData = getDefaultValueData(method);
 		if (elementValueData == null) return null;
 		ByteBuffer buf = ByteBuffer.wrap(elementValueData);
@@ -103,12 +106,12 @@ public static Annotation[] parseAnnotations(byte[] annotationsData, Class clazz)
 						clazz));
 	}
 
-	private static native byte[] getAnnotationsData(java.lang.reflect.Field field);
-	private static native byte[] getAnnotationsData(java.lang.reflect.Constructor constructor);
-	private static native byte[] getAnnotationsData(java.lang.reflect.Method method);
-	private static native byte[] getParameterAnnotationsData(java.lang.reflect.Constructor constructor);
-	private static native byte[] getParameterAnnotationsData(java.lang.reflect.Method method);
-	private static native byte[] getDefaultValueData(java.lang.reflect.Method method);
+	private static native byte[] getAnnotationsData(Field field);
+	private static native byte[] getAnnotationsData(Constructor constructor);
+	private static native byte[] getAnnotationsData(Method method);
+	private static native byte[] getParameterAnnotationsData(Constructor constructor);
+	private static native byte[] getParameterAnnotationsData(Method method);
+	private static native byte[] getDefaultValueData(Method method);
 	static native ConstantPool getConstantPool(Class clazz);
 	private static native byte[] getAnnotationsDataImpl(java.lang.Class clazz);