feat: Support App Engine Standard Java8 (#70)

* Statically link the agent .so file with libstdc++ so the agent is able to load in the Java 8 App Engine
Standard environment.
* Handle split jars since the GAE Java8 environment has a 32M file size limit 
  * Support loading cdbg java agent internals jar file when it's split into multiple jars
  * Add the jar splitter utility and build a custom java 8 package with the internals jar file split
* Add the Java 8 WEB-INF/lib and WEB-INF/classes to be auto indexed (setting extra_classpath is not an option as prior to deployment users cannot know the directory path their application will run from.

Author: jasonborg

.gitignore

@@ -12,10 +12,7 @@
 # itself. If this gets fixed at some point, these rules can be removed.
 dist/
 src/agent/antlrgen
-src/agent/internals/target
-src/agent/internals-class-loader/target/
 src/agent/internals_class_loader_static_defs.inl
-src/agent/service-account-auth/target
 src/agent/version.txt
 src/codegen/target
 src/codegen/*.cc
@@ -24,3 +21,6 @@ third_party/gflags*
 third_party/glog*
 third_party/install
 third_party/jsoncpp*
+
+# Maven build output
+target/

WORKSPACE

@@ -37,6 +37,7 @@ maven_install(
         "org.ow2.asm:asm-commons:9.1",
         "org.ow2.asm:asm-util:9.1",
         "org.yaml:snakeyaml:1.32",
+        "pl.pragmatists:JUnitParams:1.1.1",
     ],
     repositories = [
         "https://maven.google.com",

src/agent/Makefile

@@ -48,7 +48,10 @@ JAVAC=javac -source 1.7 -target 1.7
 BUILD_TARGET_PATH ?= .
 
 OPT_FLAGS = -O3 -D NDEBUG
-LDFLAGS += -shared
+
+# Statically linking libstd++ helps ensure cdbg_java_agent.so will load in a
+# wider set of environments
+LDFLAGS += -shared -static-libstdc++
 
 ifneq ($(OS),Darwin)
 # Pass these linker flags only when building with gcc (i.e., not with clang).
@@ -61,6 +64,7 @@ ANTLR_TOOL_PATH = $(THIRD_PARTY_PATH)/antlr
 ANTLR_CPP_LIB = $(THIRD_PARTY_PATH)/antlr/lib/cpp
 ANTLR_CPP_LIB_SRC = $(ANTLR_CPP_LIB)/src
 ANTLR_CPP_LIB_INCLUDE = $(ANTLR_CPP_LIB)
+JAR_SPLITTER_TOOL_PATH = $(THIRD_PARTY_PATH)/jar_splitter
 
 INCLUDES = \
 	-I$(ROOT) \
@@ -91,7 +95,9 @@ INTERNALS_CLASS_LOADER = cdbg_java_agent_internals_loader.class
 INTERNALS_CLASS_LOADER_STATIC_DEFS = internals_class_loader_static_defs.inl
 INTERNALS_JAR = internals.jar
 SERVICE_ACCOUNT_AUTH_TOOL = $(BUILD_TARGET_PATH)/cdbg_service_account_auth.jar
+JAR_SPLITTER_TOOL = $(BUILD_TARGET_PATH)/cdbg_jar_splitter.jar
 TARGET_TAR_GZ_APPENGINE = $(BUILD_TARGET_PATH)/cdbg_java_agent.tar.gz
+TARGET_TAR_GZ_APPENGINE_JAVA8 = $(BUILD_TARGET_PATH)/cdbg_java_agent_gae_java8.tar.gz
 TARGET_TAR_GZ_GCE = $(BUILD_TARGET_PATH)/cdbg_java_agent_gce.tar.gz
 TARGET_TAR_GZ_SERVICE_ACCOUNT = $(BUILD_TARGET_PATH)/cdbg_java_agent_service_account.tar.gz
 TARGET_VERSION_TXT = $(BUILD_TARGET_PATH)/version.txt
@@ -190,6 +196,7 @@ GOOGLE_API_CLIENT_JARS_CLASS_PATH := $(subst $(SPACE),:,$(GOOGLE_API_CLIENT_JARS
 
 all: \
 	packages \
+	jar_splitter_tool \
 	service_account_auth_tool \
 	target_version_txt \
 
@@ -255,6 +262,10 @@ else
 	jar cvf $(INTERNALS_JAR) -C internals/target/ .
 endif
 
+jar_splitter_tool:
+	cd $(JAR_SPLITTER_TOOL_PATH) ; mvn clean install
+	cp $(JAR_SPLITTER_TOOL_PATH)/target/cdbg_jar_splitter_tool-SNAPSHOT-jar-with-dependencies.jar $(JAR_SPLITTER_TOOL)
+
 # Separate tool to obtain OAuth access token given service account email
 # and p12 file. This tool is used by GCE bootstrap script when service account
 # authentication is enabled.
@@ -279,9 +290,12 @@ target_version_txt: internals_jar
 #    (default and recommended solution)
 # 3. Java Cloud Debugger package used on GCE with service account authentication
 #    (Identical to #2, retained for legacy purposes).
+# 4. Java Cloud Debugger package used on App Engine Standard Java 8
 packages: target_agent internals_jar $(APPENGINE_FORMAT_SCRIPT) target_version_txt
 	cp $(INTERNALS_JAR) cdbg_java_agent_internals.jar
+	java -jar $(JAR_SPLITTER_TOOL) cdbg_java_agent_internals.jar .
 	cp $(TARGET_VERSION_TXT) version.txt
 	tar cfz $(TARGET_TAR_GZ_APPENGINE) $(TARGET_AGENT) cdbg_java_agent_internals.jar $(APPENGINE_FORMAT_SCRIPT) version.txt
+	tar cfz $(TARGET_TAR_GZ_APPENGINE_JAVA8) $(TARGET_AGENT) cdbg_java_agent_internals-*.jar version.txt
 	tar cfz $(TARGET_TAR_GZ_GCE) $(TARGET_AGENT) cdbg_java_agent_internals.jar version.txt
 	tar cfz $(TARGET_TAR_GZ_SERVICE_ACCOUNT) $(TARGET_AGENT) cdbg_java_agent_internals.jar version.txt

src/agent/internals-class-loader/src/main/java/com/google/devtools/cdbg/debuglets/java/InternalsClassLoader.java

@@ -30,10 +30,12 @@ final class InternalsClassLoader extends URLClassLoader {
   /**
    * Class constructor.
    *
-   * @param internalsJarPath required path to cdbg_java_agent_internals.jar file
+   * @param internalsJarPaths required paths to cdbg_java_agent_internals.jar file. This is an array
+   * as under AppEngine Standard Java 8, the jar file is expected to be split into multiple files to
+   * work around a maximum file size limit of 32MB.
    */
-  public InternalsClassLoader(String internalsJarPath) throws MalformedURLException {
-    super(new URL[] {new URL("file:" + internalsJarPath)}, null); // Parent class loader
+  public InternalsClassLoader(String[] internalsJarPaths) throws MalformedURLException {
+    super(convertJarPathsToURLs(internalsJarPaths), null); // Parent class loader
   }
 
   @Override
@@ -47,4 +49,13 @@ protected synchronized Class<?> loadClass(String name, boolean resolve)
 
     return super.loadClass(name, resolve);
   }
+
+  static URL[] convertJarPathsToURLs(String[] internalsJarPath) throws MalformedURLException {
+    URL[] urls = new URL[internalsJarPath.length];
+    for (int i = 0; i < urls.length; ++i) {
+      urls[i] = new URL("file:" + internalsJarPath[i]);
+    }
+
+    return urls;
+  }
 }

src/agent/internals-class-loader/src/test/java/com/google/devtools/cdbg/debuglets/java/InternalsClassLoaderTest.java

@@ -81,7 +81,7 @@ public void setUp() throws MalformedURLException {
         TestDataPath.get(
             "__main__/src/agent/internals-class-loader/src/test/java/com/google/devtools/cdbg/debuglets/java",
             "InternalsClassLoaderTest.jar");
-    classLoader = new InternalsClassLoader(testsJarFile.getAbsolutePath());
+    classLoader = new InternalsClassLoader(new String[]{testsJarFile.getAbsolutePath()});
   }
 
   @Test

src/agent/internals/src/main/java/com/google/devtools/cdbg/debuglets/java/BUILD

@@ -127,6 +127,7 @@ java_library(
     ],
     deps = [
         ":datatype_converter",
+        ":gcp_environment",
         ":logger",
         ":safe_caller",
         ":yaml_config_parser",

src/agent/internals/src/main/java/com/google/devtools/cdbg/debuglets/java/ClassPathLookup.java

@@ -78,17 +78,19 @@ final class ClassPathLookup {
    *     indexed
    * @param extraClassPath optional file system locations to search for .class and .jar files beyond
    *     what's normally specified in Java class path
+   * @param agentDir The directory the agent .so file is located.
    */
-  public ClassPathLookup(boolean useDefaultClassPath, String[] extraClassPath) {
+  public ClassPathLookup(boolean useDefaultClassPath, String[] extraClassPath, String agentDir) {
     infofmt(
-        "Initializing ClassPathLookup, default classpath: %b, extra classpath: %s",
-        useDefaultClassPath, (extraClassPath == null) ? "<null>" : Arrays.asList(extraClassPath));
+        "Initializing ClassPathLookup, default classpath: %b, extra classpath: %s, agent dir: %s",
+        useDefaultClassPath, (extraClassPath == null) ? "<null>" : Arrays.asList(extraClassPath),
+        agentDir);
 
     // Try to guess where application classes might be besides class path. We only do it if the
     // debugger uses default configuration. If the user sets additional directories, we use that and
     // not try to guess anything.
     if (useDefaultClassPath && ((extraClassPath == null) || (extraClassPath.length == 0))) {
-      extraClassPath = findExtraClassPath();
+      extraClassPath = findExtraClassPath(agentDir);
     }
 
     this.useDefaultClassPath = useDefaultClassPath;
@@ -382,10 +384,14 @@ private static InputStream getResource(Class<?> cls) throws ClassNotFoundExcepti
    * in the default ROOT directory.
    *
    * <p>This function is marked as public for unit tests.
+   *
+   * @param agentDir The directory the agent .so file is located.
    */
-  public static String[] findExtraClassPath() {
+  public static String[] findExtraClassPath(String agentDir) {
     Set<String> paths = new HashSet<>();
 
+    addAppEngineJava8Paths(paths, agentDir);
+
     // Tomcat.
     addSystemPropertyRelative(paths, "catalina.base", "webapps/ROOT/WEB-INF/lib");
     addSystemPropertyRelative(paths, "catalina.base", "webapps/ROOT/WEB-INF/classes");
@@ -399,6 +405,23 @@ public static String[] findExtraClassPath() {
     return paths.toArray(new String[0]);
   }
 
+  /**
+   * If the environment is Java8 App Engine Standard it will attempt to add the USER_DIR/WEB-INF/lib
+   * and USER_DIR/WEB-INF/classes directories to the paths for indexing.
+   *
+   * <p>No effect if the environment is not Java8 App Engine Standard or the directories don't
+   * exist.
+   */
+  public static void addAppEngineJava8Paths(Set<String> paths, String agentDir) {
+    String userDir = GcpEnvironment.tryGetAppEngineJava8UserDir(agentDir);
+    if (userDir == null) {
+      return;
+    }
+
+    addPathRelative(paths, userDir, "WEB-INF/lib");
+    addPathRelative(paths, userDir, "WEB-INF/classes");
+  }
+
   /**
    * Appends a path relative to the base path defined in a system property.
    *
@@ -410,11 +433,21 @@ private static void addSystemPropertyRelative(Set<String> paths, String name, St
       return;
     }
 
-    File path = new File(value, suffix);
+    addPathRelative(paths, value, suffix);
+  }
+
+  /**
+   * Appends a path relative to a base path.
+   *
+   * <p>No effect if the combined path does not exist.
+   */
+  private static void addPathRelative(Set<String> paths, String basePath, String suffix) {
+    File path = new File(basePath, suffix);
     if (!path.exists()) {
       return;
     }
 
+    infofmt("Adding path %s for indexing", path.toString());
     paths.add(path.toString());
   }
 

src/agent/internals/src/main/java/com/google/devtools/cdbg/debuglets/java/GcpEnvironment.java

@@ -15,6 +15,8 @@
 
 import static com.google.devtools.cdbg.debuglets.java.AgentLogger.info;
 
+import java.io.File;
+
 // Needed to wrap the calls to the native logging methods which may fail for unit tests of classes
 // that use the AgentLogger, since there won't be a .so loaded that provides the native methods.
 import java.lang.UnsatisfiedLinkError;
@@ -173,6 +175,41 @@ static Map<String, String> getDebuggeeLabels() {
     return labels;
   }
 
+  /**
+   * Returns the user dir of a Java8 App Engine Standard instance. This is the base directory where
+   * the user's files are deployed. If no Java 8 App Engine Standard user dir could be determined
+   * null is returned.  It is safe to call this regardless of environment, as long as the return
+   * value is checked.
+   *
+   * <p> While there is a system property defined, "user.dir", in Java8 App Engine environments, it
+   * is unfortunately not set at the point in time this agent code runs, so it cannot be relied
+   * upon. Here we use the heuristic that the WEB-INF directory is deployed at the root of the user
+   * dir, and the deployed agent is deployed somewhere under the user dir. This means we can search
+   * through the agent directory's ancestors looking for the WEB-INFO directory, and once found
+   * we've found the user dir.
+   *
+   * @return the Java 8 App Engine Standard user directory if it was able to be determined, null
+   * otherwise.
+   */
+  public static String tryGetAppEngineJava8UserDir(String agentDir) {
+    String gaeRuntime = environmentStore.get("GAE_RUNTIME");
+    if (gaeRuntime == null || !gaeRuntime.equals("java8")) {
+      return null;
+    }
+
+    File currentDir = new File(agentDir);
+
+    while (currentDir != null) {
+      if (new File(currentDir.getAbsolutePath(), "WEB-INF").exists()) {
+        break;
+      }
+
+      currentDir = currentDir.getParentFile();
+    }
+
+    return currentDir == null ? null : currentDir.getAbsolutePath();
+  }
+
   /**
    * Gets the Debuggee.canaryMode from the corresponding System properties.
    */

src/agent/internals/src/test/java/com/google/devtools/cdbg/debuglets/java/BUILD

@@ -63,6 +63,7 @@ java_test(
     deps = [
         ":BreakpointTag",
         ":test_data_path",
+        "//src/agent/internals/src/main/java/com/google/devtools/cdbg/debuglets/java:gcp_environment",
         "//src/agent/internals/src/main/java/com/google/devtools/cdbg/debuglets/java:mapper",
         "//tests/agent:libtest_logger.so",
         "@maven//:com_google_guava_guava",
@@ -135,6 +136,7 @@ java_test(
         "//src/agent/internals/src/main/java/com/google/devtools/cdbg/debuglets/java:labels",
         "@maven//:com_google_truth_truth",
         "@maven//:junit_junit",
+        "@maven//:pl_pragmatists_JUnitParams",
     ],
 )