From 0025518c01c7ca37b32685b3197f6b9b46579f7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mat=C4=9Bj=C4=8Dek?= Date: Sun, 23 Nov 2025 12:34:15 +0100 Subject: [PATCH 01/10] Rewritten resolution of the starting process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added reporting important informations - More useful error management - --terse limits the amount of printed information - The fact that the process is alive, doesn't always mean it will be alive one second later. Check admin port too, by default. This will be configurable later. - User can configure "definition" of the started server - When process dies before the command, it is always an error - Added explicit option to print the process output - StartServerShutdownHook logging - it is kept, but the startup logging done by the AsadminMain is supported also on Windows Signed-off-by: David Matějček --- .../main/admin/test/StartServITest.java | 45 ++- .../admin/test/progress/JobManagerITest.java | 2 +- .../admin/test/rest/LoggingRestITest.java | 8 +- .../enterprise/admin/launcher/GFLauncher.java | 1 - nucleus/admin/server-mgmt/pom.xml | 5 + .../cli/ChangeAdminPasswordCommand.java | 30 +- .../cli/ChangeMasterPasswordCommand.java | 27 +- .../servermgmt/cli/DeleteDomainCommand.java | 4 +- .../servermgmt/cli/LocalServerCommand.java | 221 ++++++++---- .../servermgmt/cli/LocalStrings.properties | 13 - .../servermgmt/cli/RestartDomainCommand.java | 54 ++- .../servermgmt/cli/ServerLifeSignCheck.java | 108 ++++++ .../servermgmt/cli/ServerLifeSignChecker.java | 324 +++++++++++++++++ .../servermgmt/cli/StartDomainCommand.java | 285 +++++++-------- .../servermgmt/cli/StartServerCommand.java | 9 +- .../servermgmt/cli/StartServerHelper.java | 333 ++++++++++-------- .../servermgmt/cli/StopDomainCommand.java | 32 +- .../servermgmt/cli/StartServerHelperTest.java | 44 +++ .../cluster/CreateLocalInstanceCommand.java | 5 +- .../admin/cli/cluster/LocalStrings.properties | 1 - .../cluster/RestartLocalInstanceCommand.java | 13 +- .../cluster/StartLocalInstanceCommand.java | 167 ++++----- .../cli/cluster/StopLocalInstanceCommand.java | 10 - .../universal/process/ProcessUtils.java | 20 +- .../universal/xml/MiniXmlParser.java | 7 +- .../sun/enterprise/util/io/ServerDirs.java | 7 +- .../com/sun/enterprise/util/net/NetUtils.java | 32 +- .../embedded/AutoDisposableGlassFish.java | 3 + .../enterprise/v3/admin/StartServerHook.java | 12 +- 29 files changed, 1196 insertions(+), 626 deletions(-) create mode 100644 nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignCheck.java create mode 100644 nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignChecker.java create mode 100644 nucleus/admin/server-mgmt/src/test/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelperTest.java diff --git a/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/StartServITest.java b/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/StartServITest.java index 5d3d56b18c0..7556074d4c2 100644 --- a/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/StartServITest.java +++ b/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/StartServITest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 Eclipse Foundation and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2025 Contributors to the Eclipse Foundation. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v. 2.0, which is available at @@ -13,6 +13,7 @@ * * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 */ + package org.glassfish.main.admin.test; import java.util.stream.Stream; @@ -22,6 +23,7 @@ import org.glassfish.main.itest.tools.asadmin.AsadminResult; import org.glassfish.main.itest.tools.asadmin.StartServ; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.extension.ExtensionContext; @@ -54,29 +56,34 @@ static void setupDomain() { } } + @AfterEach + void stopDomain() { + ASADMIN.exec("stop-domain", STARTSERV_DOMAIN_NAME); + } + + @AfterAll + static void deleteDomain() { + AsadminResult result = ASADMIN.exec("list-domains"); + if (result.getOutput().contains(STARTSERV_DOMAIN_NAME)) { + ASADMIN.exec("delete-domain", STARTSERV_DOMAIN_NAME); + } + } + @ParameterizedTest @ArgumentsSource(StartServArgumentsProvider.class) public void startServerInForeground(StartServ startServ) { - try { - AsadminResult result = startServ.withTextToWaitFor("Total startup time including CLI").exec(STARTSERV_DOMAIN_NAME); - assertThat(result, asadminOK()); - } finally { - ASADMIN.exec("stop-domain", STARTSERV_DOMAIN_NAME); - } + AsadminResult result = startServ.withTextToWaitFor("Total startup time including CLI").exec(STARTSERV_DOMAIN_NAME); + assertThat(result, asadminOK()); } @ParameterizedTest @ArgumentsSource(StartServArgumentsProvider.class) @DisabledOnOs(value = WINDOWS, disabledReason = "startserv.bat is just trivial and doesn't give the error output") public void reportCorrectErrorIfAlreadyRunning(StartServ startServ) { - try { - AsadminResult result = startServ.withTextToWaitFor("Total startup time including CLI").exec(STARTSERV_DOMAIN_NAME); - assertThat(result, asadminOK()); - result = startServ.withNoTextToWaitFor().exec(STARTSERV_DOMAIN_NAME); - assertThat(result.getStdErr(), containsString("There is a process already using the admin port")); - } finally { - ASADMIN.exec("stop-domain", STARTSERV_DOMAIN_NAME); - } + AsadminResult result = startServ.withTextToWaitFor("Total startup time including CLI").exec(STARTSERV_DOMAIN_NAME); + assertThat(result, asadminOK()); + result = startServ.withNoTextToWaitFor().exec(STARTSERV_DOMAIN_NAME); + assertThat(result.getStdErr(), containsString("There is a process already using the admin port")); } @ParameterizedTest @@ -86,14 +93,6 @@ public void reportCorrectErrorIfInvalidCommand(StartServ startServ) { assertThat(result.getStdErr(), containsString("There is no such domain directory")); } - @AfterAll - static void deleteDomain() { - AsadminResult result = ASADMIN.exec("list-domains"); - if (result.getOutput().contains(STARTSERV_DOMAIN_NAME)) { - ASADMIN.exec("delete-domain", STARTSERV_DOMAIN_NAME); - } - } - static class StartServArgumentsProvider implements ArgumentsProvider { @Override diff --git a/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/progress/JobManagerITest.java b/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/progress/JobManagerITest.java index 500542a94bf..a347e377319 100644 --- a/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/progress/JobManagerITest.java +++ b/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/progress/JobManagerITest.java @@ -63,7 +63,7 @@ public void jobSurvivesRestart() throws Exception { "--cleanup-initial-delay=0s", "--cleanup-poll-interval=0s"), asadminOK()); assertThat(ASADMIN.exec(COMMAND_PROGRESS_SIMPLE), asadminOK()); - assertThat(ASADMIN.exec("restart-domain", "--timeout", "60"), asadminOK()); + assertThat(ASADMIN.exec("restart-domain", "--timeout", "60", "domain1"), asadminOK()); assertThat(ASADMIN.exec("list-jobs").getStdOut(), stringContainsInOrder(COMMAND_PROGRESS_SIMPLE, "COMPLETED")); JobTestExtension.doAndDisableJobCleanup(); } diff --git a/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/rest/LoggingRestITest.java b/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/rest/LoggingRestITest.java index ac69928bd3c..75d41f677b4 100644 --- a/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/rest/LoggingRestITest.java +++ b/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/rest/LoggingRestITest.java @@ -31,6 +31,8 @@ import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; import static org.glassfish.main.itest.tools.GlassFishTestEnvironment.getAsadmin; import static org.glassfish.main.itest.tools.asadmin.AsadminResultMatcher.asadminOK; +import static org.hamcrest.CoreMatchers.anyOf; +import static org.hamcrest.CoreMatchers.startsWith; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyOrNullString; @@ -39,7 +41,6 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.startsWith; import static org.junit.jupiter.api.Assertions.assertAll; /** @@ -55,7 +56,7 @@ public class LoggingRestITest extends RestTestBase { public static void fillUpLog() { // The server log may become empty due to log rotation. // Restart domain to fill it up. - AsadminResult result = getAsadmin().exec("restart-domain", "--timeout", "60"); + AsadminResult result = getAsadmin().exec("restart-domain", "--timeout", "60", "domain1"); assertThat(result, asadminOK()); } @@ -100,7 +101,8 @@ public void logFileNames() throws Exception { // Depends on the order of tests, there may be rolled file too. assertAll( () -> assertThat("InstanceLogFileNames", logFileNames.length(), greaterThanOrEqualTo(1)), - () -> assertThat(logFileNames.get(0).toString(), startsWith("server.log")) + () -> assertThat(logFileNames.get(0).toString(), + anyOf(startsWith("restart.log"), startsWith("server.log"))) ); } diff --git a/nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java b/nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java index 22a94f4bd9c..d55e20883c0 100644 --- a/nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java +++ b/nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java @@ -382,7 +382,6 @@ public String getLogFilename() { throw new IllegalStateException("Call to getLogFilename() before it has been initialized!"); } - /** * @return null or a port number */ diff --git a/nucleus/admin/server-mgmt/pom.xml b/nucleus/admin/server-mgmt/pom.xml index faf424536b8..282fcd595a0 100644 --- a/nucleus/admin/server-mgmt/pom.xml +++ b/nucleus/admin/server-mgmt/pom.xml @@ -56,6 +56,11 @@ glassfish-jdk-extensions ${project.version} + + org.glassfish.main + glassfish-jul-extension + compile + org.glassfish.main.admin admin-util diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ChangeAdminPasswordCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ChangeAdminPasswordCommand.java index c35caedfd89..ae368448d96 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ChangeAdminPasswordCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ChangeAdminPasswordCommand.java @@ -172,9 +172,8 @@ private int changeAdminPasswordLocally(String domainDir, String domainName) thro throw new CommandException(strings.get("CannotExecuteLocally")); } - GFLauncher launcher = null; try { - launcher = GFLauncherFactory.getInstance(RuntimeType.DAS); + GFLauncher launcher = GFLauncherFactory.getInstance(RuntimeType.DAS); GFLauncherInfo info = launcher.getInfo(); info.setDomainName(domainName); info.setDomainParentDir(domainDir); @@ -183,31 +182,28 @@ private int changeAdminPasswordLocally(String domainDir, String domainName) thro //If secure admin is enabled and if new password is null //throw new exception if (launcher.isSecureAdminEnabled()) { - if ((newpassword == null) || (newpassword.isEmpty())) { + if (newpassword == null || newpassword.isEmpty()) { throw new CommandException(strings.get("NullNewPassword")); } } String adminKeyFile = launcher.getAdminRealmKeyFile(); - if (adminKeyFile != null) { - //This is a FileRealm, instantiate it. - FileRealmHelper helper = new FileRealmHelper(adminKeyFile); - - //Authenticate the old password - String[] groups = helper.authenticate(programOpts.getUser(), password.toCharArray()); - if (groups == null) { - throw new CommandException(strings.get("InvalidCredentials", programOpts.getUser())); - } - helper.updateUser(programOpts.getUser(), programOpts.getUser(), newpassword.toCharArray(), null); - helper.persist(); - return SUCCESS; - - } else { + if (adminKeyFile == null) { //Cannot change password locally for non file realms throw new CommandException(strings.get("NotFileRealmCannotChangeLocally")); + } + //This is a FileRealm, instantiate it. + FileRealmHelper helper = new FileRealmHelper(adminKeyFile); + //Authenticate the old password + String[] groups = helper.authenticate(programOpts.getUser(), password.toCharArray()); + if (groups == null) { + throw new CommandException(strings.get("InvalidCredentials", programOpts.getUser())); } + helper.updateUser(programOpts.getUser(), programOpts.getUser(), newpassword.toCharArray(), null); + helper.persist(); + return SUCCESS; } catch (MiniXmlParserException ex) { throw new CommandException(ex); diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ChangeMasterPasswordCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ChangeMasterPasswordCommand.java index be6677b38ed..f79490d27d6 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ChangeMasterPasswordCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ChangeMasterPasswordCommand.java @@ -1,4 +1,5 @@ /* + * Copyright (c) 2025 Contributors to the Eclipse Foundation. * Copyright (c) 1997, 2018 Oracle and/or its affiliates. All rights reserved. * * This program and the accompanying materials are made available under the @@ -92,21 +93,19 @@ protected int executeCommand() throws CommandException { if (nodeDir != null) { command = CLICommand.getCommand(habitat, CHANGE_MASTER_PASSWORD_NODE); return command.execute(argv); - } else { - - // nodeDir is not specified and domainNameOrNodeName is not a domain. - // It could be a node - // We add defaultNodeDir parameter to args - ArrayList arguments = new ArrayList(Arrays.asList(argv)); - arguments.remove(argv.length - 1); - arguments.add("--nodedir"); - arguments.add(getDefaultNodesDirs().getAbsolutePath()); - arguments.add(domainNameOrNodeName); - String[] newargs = (String[]) arguments.toArray(new String[arguments.size()]); - - command = CLICommand.getCommand(habitat, CHANGE_MASTER_PASSWORD_NODE); - return command.execute(newargs); } + // nodeDir is not specified and domainNameOrNodeName is not a domain. + // It could be a node + // We add defaultNodeDir parameter to args + ArrayList arguments = new ArrayList<>(Arrays.asList(argv)); + arguments.remove(argv.length - 1); + arguments.add("--nodedir"); + arguments.add(getDefaultNodesDirs().getAbsolutePath()); + arguments.add(domainNameOrNodeName); + String[] newargs = arguments.toArray(String[]::new); + + command = CLICommand.getCommand(habitat, CHANGE_MASTER_PASSWORD_NODE); + return command.execute(newargs); } catch (IOException e) { throw new CommandException(e.getMessage(), e); } diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/DeleteDomainCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/DeleteDomainCommand.java index 9ad2fc76dcd..aba38e21953 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/DeleteDomainCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/DeleteDomainCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2024 Contributors to the Eclipse Foundation + * Copyright (c) 2022, 2025 Contributors to the Eclipse Foundation * Copyright (c) 1997, 2018 Oracle and/or its affiliates. All rights reserved. * * This program and the accompanying materials are made available under the @@ -81,7 +81,7 @@ private void checkRunning() throws CommandException { /** * Check that the domain directory can be renamed, to increase the likelyhood that it can be deleted. */ - private void checkRename() throws CommandException { + private void checkRename() { boolean ok = true; try { File root = getDomainsDir(); diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java index e8811caabb6..6933f331f5a 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java @@ -21,6 +21,8 @@ import com.sun.enterprise.admin.cli.CLIConstants; import com.sun.enterprise.admin.cli.ProgramOptions; import com.sun.enterprise.admin.cli.remote.RemoteCLICommand; +import com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignChecker.GlassFishProcess; +import com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignChecker.ServerLifeSigns; import com.sun.enterprise.security.store.PasswordAdapter; import com.sun.enterprise.universal.io.SmartFile; import com.sun.enterprise.universal.process.ProcessUtils; @@ -31,10 +33,12 @@ import java.io.File; import java.io.IOException; +import java.lang.System.Logger; +import java.lang.System.Logger.Level; +import java.nio.file.Files; import java.time.Duration; import java.util.List; import java.util.function.Supplier; -import java.util.logging.Level; import org.glassfish.api.ActionReport; import org.glassfish.api.ActionReport.ExitCode; @@ -44,14 +48,17 @@ import static com.sun.enterprise.admin.cli.CLIConstants.DEFAULT_ADMIN_PORT; import static com.sun.enterprise.admin.cli.CLIConstants.DEFAULT_HOSTNAME; import static com.sun.enterprise.admin.cli.ProgramOptions.PasswordLocation.LOCAL_PASSWORD; +import static com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignChecker.step; +import static com.sun.enterprise.universal.process.ProcessUtils.loadPid; +import static com.sun.enterprise.universal.process.ProcessUtils.waitForNewPid; +import static com.sun.enterprise.universal.process.ProcessUtils.waitWhileIsAlive; import static com.sun.enterprise.util.SystemPropertyConstants.KEYSTORE_PASSWORD_DEFAULT; import static com.sun.enterprise.util.SystemPropertyConstants.MASTER_PASSWORD_ALIAS; import static com.sun.enterprise.util.SystemPropertyConstants.MASTER_PASSWORD_FILENAME; import static com.sun.enterprise.util.SystemPropertyConstants.MASTER_PASSWORD_PASSWORD; import static com.sun.enterprise.util.SystemPropertyConstants.TRUSTSTORE_FILENAME_DEFAULT; -import static java.util.logging.Level.CONFIG; -import static java.util.logging.Level.FINER; -import static java.util.logging.Level.FINEST; +import static java.lang.System.Logger.Level.DEBUG; +import static java.lang.System.Logger.Level.INFO; /** * A class that's supposed to capture all the behavior common to operation on a "local" server. @@ -60,6 +67,8 @@ */ public abstract class LocalServerCommand extends CLICommand { + private static final Logger LOG = System.getLogger(LocalServerCommand.class.getName()); + private ServerDirs serverDirs; /** @@ -132,6 +141,7 @@ private HostAndPort findReachableAdminAddress(String userHost, Integer userPort, Integer port = userPort == null ? DEFAULT_ADMIN_PORT : userPort; boolean secure = userSecure == null ? false : userSecure; HostAndPort endpoint = new HostAndPort(host, port, secure); + LOG.log(DEBUG, () -> "Checking candidate: " + endpoint); return ProcessUtils.isListening(endpoint) ? endpoint : null; } for (HostAndPort candidate : adminEndpointCandidates) { @@ -139,6 +149,7 @@ private HostAndPort findReachableAdminAddress(String userHost, Integer userPort, final int port = userPort == null ? candidate.getPort() : userPort; final boolean secure = userSecure == null ? candidate.isSecure() : userSecure; HostAndPort endpoint = new HostAndPort(host, port, secure); + LOG.log(DEBUG, () -> "Checking candidate: " + endpoint); if (ProcessUtils.isListening(endpoint)) { return endpoint; } @@ -164,6 +175,10 @@ protected final List loadAdminAddresses(File domainXml, String serv } + protected final HostAndPort getUserProvidedAdminAddress() { + return new HostAndPort(programOpts.getHost(), programOpts.getPort(), programOpts.isSecure()); + } + protected final void setServerDirs(ServerDirs sd) { serverDirs = sd; } @@ -183,7 +198,7 @@ protected final boolean isLocal() { protected final void setLocalPassword() { String pw = serverDirs == null ? null : serverDirs.getLocalPassword(); programOpts.setPassword(pw == null ? null : pw.toCharArray(), LOCAL_PASSWORD); - logger.finer(ok(pw) ? "Using local password" : "Not using local password"); + LOG.log(DEBUG, () -> ok(pw) ? "Using local password" : "Not using local password"); } protected final void unsetLocalPassword() { @@ -223,7 +238,7 @@ protected final String readFromMasterPasswordFile() { PasswordAdapter pw = new PasswordAdapter(mpf.getAbsolutePath(), MASTER_PASSWORD_PASSWORD.toCharArray()); return pw.getPasswordForAlias(MASTER_PASSWORD_ALIAS); } catch (Exception e) { - logger.log(Level.WARNING, "A master password file reading error: " + e.toString(), e); + LOG.log(Level.WARNING, "A master password file reading error: " + e.toString(), e); return null; } } @@ -233,7 +248,7 @@ protected final boolean verifyMasterPassword(String mpv) { } protected boolean loadAndVerifyKeystore(File jks, String mpv) { - logger.log(FINEST, "loading keystore: " + jks); + LOG.log(DEBUG, "loading keystore: " + jks); if (jks == null || mpv == null) { return false; } @@ -241,7 +256,7 @@ protected boolean loadAndVerifyKeystore(File jks, String mpv) { new KeyTool(jks, mpv.toCharArray()).loadKeyStore(); return true; } catch (Exception e) { - logger.log(FINER, e.getMessage(), e); + LOG.log(DEBUG, e.getMessage(), e); return false; } } @@ -254,25 +269,25 @@ protected final String getMasterPassword() throws CommandException { // Yes, returning master password as a string is not right ... final int countOfRetries = 3; final long start = System.currentTimeMillis(); - String mpv = passwords.get(CLIConstants.MASTER_PASSWORD); - if (mpv == null) { + String masterPassword = passwords.get(CLIConstants.MASTER_PASSWORD); + if (masterPassword == null) { // not specified in the password file // optimization for the default case - mpv = KEYSTORE_PASSWORD_DEFAULT; - if (!verifyMasterPassword(mpv)) { - mpv = readFromMasterPasswordFile(); - if (!verifyMasterPassword(mpv)) { - mpv = retry(countOfRetries); + masterPassword = KEYSTORE_PASSWORD_DEFAULT; + if (!verifyMasterPassword(masterPassword)) { + masterPassword = readFromMasterPasswordFile(); + if (!verifyMasterPassword(masterPassword)) { + masterPassword = retry(countOfRetries); } } } else { // the passwordfile contains AS_ADMIN_MASTERPASSWORD, use it - if (!verifyMasterPassword(mpv)) { - mpv = retry(countOfRetries); + if (!verifyMasterPassword(masterPassword)) { + masterPassword = retry(countOfRetries); } } - logger.log(FINER, "Time spent in master password extraction: {0} ms", (System.currentTimeMillis() - start)); - return mpv; + LOG.log(DEBUG, "Time spent in master password extraction: " + (System.currentTimeMillis() - start) + " ms"); + return masterPassword; } /** @@ -286,13 +301,13 @@ protected final boolean isThisServer(File ourDir, String directoryKey) { } ourDir = getUniquePath(ourDir); - logger.log(FINER, "Check if server is at location {0}", ourDir); + LOG.log(DEBUG, "Check if server is at location {0}", ourDir); try { RemoteCLICommand cmd = new RemoteCLICommand("__locations", programOpts, env); ActionReport report = cmd.executeAndReturnActionReport(new String[] { "__locations" }); String theirDirPath = report.findProperty(directoryKey); - logger.log(FINER, "Remote server has root directory {0}", theirDirPath); + LOG.log(DEBUG, "Remote server has root directory {0}", theirDirPath); if (ok(theirDirPath)) { File theirDir = getUniquePath(new File(theirDirPath)); @@ -311,72 +326,155 @@ protected final boolean isThisServer(File ourDir, String directoryKey) { * @return PID or null if unreachable */ protected final Long getServerPid() { - Long pidFromFile = ProcessUtils.loadPid(getServerDirs().getPidFile()); + Long pidFromFile = loadPid(getServerDirs().getPidFile()); try { RemoteCLICommand command = new RemoteCLICommand("__locations", programOpts, env); ActionReport report = command.executeAndReturnActionReport("__locations"); if (report.getActionExitCode() == ExitCode.SUCCESS) { long pidFromAdmin = Long.parseLong(report.findProperty("Pid")); if (pidFromFile == null || !pidFromFile.equals(pidFromAdmin)) { - logger.log(Level.SEVERE, - "PID should be the same: PID from file = " + pidFromFile + ", pidFromAdmin = " + pidFromAdmin); + LOG.log(Level.ERROR, "PID should be the same: PID from file = " + pidFromFile + + ", while PID received from admin endpoint = " + pidFromAdmin); } return pidFromAdmin; } return null; } catch (Exception e) { - logger.log(Level.SEVERE, "The server PID could not be resolved, sending PID from file: " + pidFromFile, e); + LOG.log(Level.ERROR, "The server PID could not be resolved, sending PID from file: " + pidFromFile + ".", e); return pidFromFile; } } + /** + * Waits until server stops + * + * @param pid + * @param adminAddress + * @param timeout can be null + * @throws CommandException if we time out. + */ + protected final void waitForStop(final Long pid, final HostAndPort adminAddress, final Duration timeout) + throws CommandException { + LOG.log(DEBUG, "waitForStop(pid={0}, oldAdminAddress={1}, timeout={2})", pid, adminAddress, timeout); + + final boolean printDots = !programOpts.isTerse(); + final Duration portTimeout; + if (pid == null) { + portTimeout = timeout; + } else { + portTimeout = step("Waiting for the death of the process with pid " + pid, timeout, + () -> waitWhileIsAlive(pid, timeout, printDots)); + if (ProcessUtils.isAlive(pid)) { + throw new CommandException("Timed out waiting for the server process to stop."); + } + } + if (adminAddress == null) { + return; + } + LOG.log(INFO, "Waiting until admin endpoint {0} is free.", adminAddress); + final boolean stopped = ProcessUtils.waitWhileListening(adminAddress, + portTimeout == null ? Duration.ofHours(1L) : portTimeout, printDots); + if (stopped) { + return; + } + throw new CommandException("Timed out waiting for the server to stop."); + } /** - * Waits until server stops and starts + * Waits until server is running - with different pid * * @param oldPid - * @param oldAdminAddress - * @param newAdminAddress new admin endpoint - usually same as old, but it could change with restart. + * @param lifeSignCheck + * @param adminEndpointsSupplier * @param timeout can be null * @throws CommandException if we time out. */ - protected final void waitForRestart(final Long oldPid, final HostAndPort oldAdminAddress, - final HostAndPort newAdminAddress, final Duration timeout) throws CommandException { - logger.log(Level.FINEST, "waitForRestart(oldPid={0}, oldAdminAddress={1}, newAdminAddress={2}, timeout={3})", - new Object[] {oldPid, oldAdminAddress, newAdminAddress, timeout}); + protected final String waitForStart(final Long oldPid, final ServerLifeSignCheck lifeSignCheck, + final Supplier> adminEndpointsSupplier, final Duration timeout) throws CommandException { + LOG.log(DEBUG, "waitForStart(oldPid={0}, adminEndpoints, timeout={1})", oldPid, timeout); final boolean printDots = !programOpts.isTerse(); - final boolean stopped = oldPid == null || ProcessUtils.waitWhileIsAlive(oldPid, timeout, printDots); - if (!stopped) { - throw new CommandException("Timed out waiting for the server to restart"); - } - logger.log(CONFIG, "Server instance is stopped, now we wait for the start on {0}", newAdminAddress); - // Could change - programOpts.setHostAndPort(newAdminAddress); - final Supplier signStart = () -> { - if (!ProcessUtils.isListening(newAdminAddress)) { - // nobody is listening - return false; - } - try { - resetServerDirs(); - setLocalPassword(); - } catch (Exception e) { - logger.log(FINEST, "The endpoint is alive, but we failed to reset the local password.", e); - return false; - } - Long newPid = ProcessUtils.loadPid(getServerDirs().getPidFile()); - if (newPid == null) { - return false; + final File pidFile = getServerDirs().getPidFile(); + final Duration startTimeout; + if (oldPid == null) { + startTimeout = timeout; + } else { + startTimeout = step("Waiting for the new PID.", timeout, + () -> waitForNewPid(oldPid, pidFile, timeout, printDots)); + } + if (startTimeout != null && startTimeout.isNegative()) { + throw new CommandException(reportPidFileIssue(pidFile)); + } + final Long pid = loadPid(getServerDirs().getPidFile()); + if (pid == null) { + throw new CommandException(reportPidFileIssue(pidFile)); + } + + try { + resetServerDirs(); + setLocalPassword(); + } catch (Exception e) { + LOG.log(Level.WARNING, "The endpoint is alive, but we failed to reset the local password.", e); + } + + LOG.log(INFO, () -> "Waiting until start of " + lifeSignCheck.getServerTitleAndName() + " completes."); + + final ServerLifeSignChecker checker = new ServerLifeSignChecker(lifeSignCheck, pidFile, adminEndpointsSupplier, printDots); + final GlassFishProcess process = GlassFishProcess.of(pid); + final ServerLifeSigns signs = checker.watchStartup(process, startTimeout); + final String report = report(signs); + if (signs.isError()) { + throw new CommandException(report); + } + return report; + } + + private String reportPidFileIssue(final File pidFile) { + File restartLogFile = serverDirs.getRestartLogFile(); + String error = "Could not load the PID number from file " + pidFile; + String restartLog = loadRestartLog(restartLogFile); + if (restartLog == null) { + return error + "\nThe " + restartLogFile + " file could not be loaded too."; + } + return error + "\n" + restartLog; + } + + private String report(ServerLifeSigns signs) { + final StringBuilder report = new StringBuilder(2048); + report.append('\n').append(signs.getSummary()); + if (signs.getSuggestion() != null) { + report.append('\n').append(signs.getSuggestion()); + } + report.append("\n Location: ").append(getServerDirs().getServerDir()); + final String situationReport = signs.getSituationReport(); + if (situationReport != null) { + report.append(signs.getSituationReport()); + } + if (signs.isError()) { + final String restartLog = loadRestartLog(serverDirs.getRestartLogFile()); + if (restartLog != null) { + report.append('\n').append(restartLog); } - logger.log(FINEST, "The server pid is {0}", newPid); - return ProcessUtils.isAlive(newPid); - }; - if (!ProcessUtils.waitFor(signStart, timeout, printDots)) { - throw new CommandException("Timed out waiting for the server to restart"); } + return report.toString(); } + private String loadRestartLog(final File logFile ) { + if (logFile == null || !logFile.exists()) { + return null; + } + try { + String report = "Found restart log file content: \n##########\n" + + Files.readString(serverDirs.getRestartLogFile().toPath()) + "\n##########\n"; + // The log is there just to diagnose failed start phase of the restart. + // We don't want to delete it if we cannot load it. + logFile.delete(); + return report; + } catch (IOException e) { + LOG.log(Level.WARNING, "Failed to read the restart log file: " + serverDirs.getRestartLogFile(), e); + return null; + } + } /** * @return uptime from the server. @@ -390,7 +488,7 @@ protected final long getUptime() throws CommandException { throw new CommandException("Server is not running, will attempt to start it..."); } - logger.log(FINER, "server uptime: {0}", up_ms); + LOG.log(DEBUG, () -> "Server uptime: " + up_ms + " ms"); return up_ms; } @@ -437,7 +535,7 @@ private File getJKS() { if (mp.canRead()) { return mp; } - logger.log(FINEST, "File does not exist or is not readable: {0}", mp); + LOG.log(DEBUG, "File does not exist or is not readable: {0}", mp); return null; } @@ -459,7 +557,6 @@ private String retry(int times) throws CommandException { String mpv; // prompt times times for (int i = 0; i < times; i++) { - // XXX - I18N String prompt = "Enter master password - (" + (times - i) + ") attempt(s) remain)> "; char[] mpvArr = super.readPassword(prompt); mpv = mpvArr != null ? new String(mpvArr) : null; diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalStrings.properties b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalStrings.properties index 3cc5834f563..4527dba2145 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalStrings.properties +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalStrings.properties @@ -136,16 +136,11 @@ database.driver.version.msg=Database Driver Version: [{0}] jdbc.version.msg=JDBC Specification Version: [{0}] CouldNotGetSysInfo=Unable to retrieve database system information. noPorts=Error in domain.xml -- no administration ports found. -WaitServer=Waiting for {0} to start -serverDied=Error starting {0}.\nThe server exited prematurely with exit code {1}. -serverDiedOutput=Error starting {0}.\nThe server exited prematurely with exit code {1}.\nBefore it died, it produced the following output:\n\n{2} remoteError=Error on remote server: {0} unknownFormat=Unknown server response. cause=Cause: {0} #start-domain and start-instance -ServerRunning=There is a process already using the admin port {0} -- it probably is another instance of a GlassFish server. -ServerStart.SuccessMessage=Successfully started the {0}: {1}\n{0} Location: {2}\nLog File: {3}\nAdmin Port: {4} DomainLocation=Started domain: {0}\nDomain location: {1}\nLog file: {2} DomainAdminPort=Admin port for the domain: {0} DomainDebugPort=Debug port for the domain: {0} @@ -173,7 +168,6 @@ commands.monitor.httplistener_detail=******************************************* ## restart-domain restart=Restarting Domain... -restartDomain.success=Successfully restarted the domain restartChangeDebug=Restarting Domain and explicitly setting debug to {0}... ## change-admin-password @@ -185,13 +179,6 @@ change.admin.password.newpassword=Enter the new admin password change.admin.password.newpassword.again=Enter the new admin password again ## LocalServerCommand -DAS=Domain Administration Server -INSTANCE=Instance Server -serverNoStart=No response from the {0} ({1}) after {2} seconds.\n\ -The command is either taking too long to complete or the server has failed.\n\ -Please see the server log files for command status. \n\ -Please start with the --verbose option in order to see early messages. -deathwait_timeout=Waited {0} milliseconds for the server to die. It did not die. Can not restart. Please kill it manually. NullNewPassword=CLI197: New password must not be null when secure-admin is enabled. NotFileRealmCannotChangeLocally=CLI198: The admin realm is not a file based realm.The admin password cannot be changed locally. CannotExecuteLocally=CLI199: This command cannot be executed locally. diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/RestartDomainCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/RestartDomainCommand.java index 76d2e2addda..10167b13f30 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/RestartDomainCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/RestartDomainCommand.java @@ -26,6 +26,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.function.Supplier; import org.glassfish.api.Param; import org.glassfish.api.admin.CommandException; @@ -36,6 +37,8 @@ import static com.sun.enterprise.admin.cli.CLIConstants.DEATH_TIMEOUT_MS; import static com.sun.enterprise.admin.cli.CLIConstants.WAIT_FOR_DAS_TIME_MS; +import static com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignChecker.step; +import static com.sun.enterprise.admin.servermgmt.cli.StartServerHelper.parseCustomEndpoints; /** * THe restart-domain command. The local portion of this command is only used to block until: @@ -55,13 +58,28 @@ @Service(name = "restart-domain") @PerLookup public class RestartDomainCommand extends StopDomainCommand { - private static final LocalStringsImpl strings = new LocalStringsImpl(RestartDomainCommand.class); + private static final LocalStringsImpl I18N = new LocalStringsImpl(RestartDomainCommand.class); @Param(name = "debug", optional = true) private Boolean debug; + // Cannot be disabled, we use it to get PID. + private boolean checkPidFile = true; + + @Param(name = "check-process-alive", optional = true, defaultValue = "true") + private boolean checkProcessAlive; + + @Param(name = "check-admin-port", optional = true, defaultValue = "true") + private boolean checkAdminEndpoint; + + @Param(name = "server-output", shortName = "o", optional = true) + private Boolean printServerOutput; + + @Param(optional = true) + private String customEndpoints; + @Inject - private ServiceLocator habitat; + private ServiceLocator serviceLocator; /** * Execute the restart-domain command. @@ -76,7 +94,7 @@ protected void doCommand() throws CommandException { + "Please stop and then restart the server - or fix the password file."); } - // Save old values before executing restart + // oldPid is received from the running server. final Long oldPid = getServerPid(); final HostAndPort oldAdminAddress = getReachableAdminAddress(); final HostAndPort newAdminEndpoint = getAdminAddress("server"); @@ -87,8 +105,16 @@ protected void doCommand() throws CommandException { cmd.executeAndReturnOutput("restart-domain", "--debug", debug.toString()); } - waitForRestart(oldPid, oldAdminAddress, newAdminEndpoint, getRestartTimeout()); - logger.info(strings.get("restartDomain.success")); + final Duration timeout = getRestartTimeout(); + final Duration startTimeout = step(null, timeout, + () -> waitForStop(isLocal() ? oldPid : null, oldAdminAddress, timeout)); + + final List userEndpoints = parseCustomEndpoints(customEndpoints); + final ServerLifeSignCheck lifeSignCheck = new ServerLifeSignCheck("domain " + getDomainName(), + printServerOutput, checkPidFile, checkProcessAlive, checkAdminEndpoint, userEndpoints); + final Supplier> adminEndpointsSupplier = () -> List.of(getReachableAdminAddress()); + final String report = waitForStart(oldPid, lifeSignCheck, adminEndpointsSupplier, startTimeout); + logger.info(report); } /** @@ -100,7 +126,7 @@ protected int dasNotRunning() throws CommandException { throw new CommandException("Remote server is not running, can not restart it"); } logger.warning("Server is not running, will attempt to start it..."); - CLICommand cmd = habitat.getService(CLICommand.class, "start-domain"); + CLICommand cmd = serviceLocator.getService(CLICommand.class, "start-domain"); /* * Collect the arguments that also apply to start-domain. * The start-domain CLICommand object will already have the @@ -130,10 +156,24 @@ protected int dasNotRunning() throws CommandException { opts.add("--timeout"); opts.add(Long.toString(startTimeout.toSeconds())); } + opts.add("--check-pid-file"); + opts.add(Boolean.toString(checkPidFile)); + opts.add("--check-process-alive"); + opts.add(Boolean.toString(checkProcessAlive)); + opts.add("--check-admin-port"); + opts.add(Boolean.toString(checkAdminEndpoint)); + if (printServerOutput != null) { + // TODO: At this moment works just when the domain was not running + opts.add("--server-output"); + opts.add(Boolean.toString(printServerOutput)); + } + if (customEndpoints != null) { + opts.add("--custom-endpoints"); + opts.add(customEndpoints); + } if (getDomainName() != null) { opts.add(getDomainName()); } - return cmd.execute(opts.toArray(String[]::new)); } diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignCheck.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignCheck.java new file mode 100644 index 00000000000..7bd643a85f6 --- /dev/null +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignCheck.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2025 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0, which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the + * Eclipse Public License v. 2.0 are satisfied: GNU General Public License, + * version 2 with the GNU Classpath Exception, which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + */ + +package com.sun.enterprise.admin.servermgmt.cli; + +import com.sun.enterprise.util.HostAndPort; + +import java.util.List; + +public class ServerLifeSignCheck { + + private final String serverTitleAndName; + private final Boolean printServerOutput; + + private final boolean pidFile; + private final boolean processAlive; + private final boolean adminEndpoint; + private final List customEndpoints; + + + + public ServerLifeSignCheck(String serverTitleAndName, Boolean printServerOutput, boolean pidFile, + boolean processAlive, boolean adminEndpoint, List customEndpoints) { + this.serverTitleAndName = serverTitleAndName; + this.printServerOutput = printServerOutput; + this.pidFile = pidFile; + this.processAlive = processAlive; + this.adminEndpoint = adminEndpoint; + this.customEndpoints = customEndpoints; + } + + + public String getServerTitleAndName() { + return serverTitleAndName; + } + + + /** + *
    + *
  • null - print on error + *
  • true - print always + *
  • false - print never + *
+ * + * @return null/true/false + */ + public Boolean getPrintServerOutput() { + return printServerOutput; + } + + + /** + * Usage: Rare, when we don't have permissions to check if the process is alive, or user uses own set of checks. + * @return true to check the pid file existence. + */ + public boolean isPidFile() { + return pidFile; + } + + + /** + * Usage: When we don't need to wait until server is listening on ports. + * @return true to check the {@link Process#isAlive()}. + */ + public boolean isProcessAlive() { + return processAlive; + } + + + /** + * Usage: When we need just admin endpoint to communicate with. + * @return true to check at least one admin endpoint loaded from domain.xml. + */ + public boolean isAdminEndpoint() { + return adminEndpoint; + } + + + /** + * Usage: When we use strict control over incoming requests, ie. requests + * @return true to check custom endpoints provided by the user. + */ + public boolean isCustomEndpoints() { + return !customEndpoints.isEmpty(); + } + + + /** + * Usage: When we use strict control over incoming requests, ie. requests + * @return custom endpoints provided by the user. + */ + public List getCustomEndpoints() { + return customEndpoints; + } +} diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignChecker.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignChecker.java new file mode 100644 index 00000000000..1149e78ea88 --- /dev/null +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignChecker.java @@ -0,0 +1,324 @@ +/* + * Copyright (c) 2025 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0, which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the + * Eclipse Public License v. 2.0 are satisfied: GNU General Public License, + * version 2 with the GNU Classpath Exception, which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + */ + +package com.sun.enterprise.admin.servermgmt.cli; + +import com.sun.enterprise.universal.process.ProcessUtils; +import com.sun.enterprise.util.HostAndPort; + +import java.io.File; +import java.lang.System.Logger; +import java.lang.System.Logger.Level; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.function.Supplier; + +import org.glassfish.api.admin.CommandException; + +import static java.lang.System.Logger.Level.INFO; + +public class ServerLifeSignChecker { + private static final Logger LOG = System.getLogger(ServerLifeSignChecker.class.getName()); + + private final ServerLifeSignCheck checks; + private final File pidFile; + private final Supplier> adminEndpointsSupplier; + private final boolean verbose; + + + public ServerLifeSignChecker(ServerLifeSignCheck checks, File pidFile, Supplier> adminEndpointsSupplier, boolean verbose) { + this.checks = checks; + this.pidFile = pidFile; + this.adminEndpointsSupplier = adminEndpointsSupplier; + this.verbose = verbose; + } + + /** + * Sign of the finished startup means that the starting process finished but it doesn't tell us + * the result. When it returns true it means that we can evaluate the final result which cannot + * change any more. + * + * @param process + * @param timeout + * @return true if we can make the final decision. + */ + public ServerLifeSigns watchStartup(GlassFishProcess process, Duration timeout) { + final ServerLifeSigns signs = new ServerLifeSigns(); + if (timeout != null && timeout.isNegative()) { + signs.situationReport = createSituationReport(process); + return createTimeoutReport(signs); + } + if (!checks.isPidFile() && !checks.isProcessAlive() && !checks.isAdminEndpoint() && !checks.isCustomEndpoints()) { + signs.summary = "All checks of the server state were disabled. Assuming the server is running."; + signs.situationReport = createSituationReport(process); + signs.suggestion = getSuggestions(); + return signs; + } + final boolean wasTimeout = !waitFor(process, timeout); + signs.situationReport = createSituationReport(process); + if (wasTimeout) { + return createTimeoutReport(signs); + } + if (!process.isAlive()) { + signs.error = true; + signs.suggestion = getSuggestions(); + final Integer exitCode = process.exitCode(); + if (exitCode == null) { + signs.summary = "The process died."; + } else { + signs.summary = "The startup command return code was " + exitCode + " which means that the start "; + if (exitCode == 0) { + signs.summary += "succeded, however later the process stopped for some reason."; + } else { + signs.summary += "failed with exit code " + exitCode + "."; + } + } + return signs; + } + signs.summary = "Successfully started the " + checks.getServerTitleAndName() + "."; + return signs; + } + + private boolean waitFor(GlassFishProcess process, Duration timeout) { + // true -> in time, false -> timeout + final Supplier signOfFinishedStartup = () -> { + if (checks.isProcessAlive()) { + // If process died, start always failed and we are done. + // however this check can be explicitly disabled. + if (!process.isAlive()) { + return true; + } + } + if (checks.isCustomEndpoints()) { + if (!isListeningOnAllEndpoints(checks.getCustomEndpoints())) { + return false; + } + } + if (checks.isAdminEndpoint()) { + if (!isListeningOnAnyEndpoint(adminEndpointsSupplier.get())) { + return false; + } + } + if (checks.isPidFile()) { + if (ProcessUtils.loadPid(pidFile) == null) { + return false; + } + } + return true; + }; + return ProcessUtils.waitFor(signOfFinishedStartup, timeout, verbose); + } + + private ServerLifeSigns createTimeoutReport(final ServerLifeSigns signs) { + signs.error = true; + signs.summary = "Failed to confirm that the server is running - timed out." + + " The command is either taking too long to complete" + + " or the startup has failed" + + " or we are not permitted to complete all checks."; + signs.suggestion = getSuggestions(); + return signs; + } + + private String createSituationReport(GlassFishProcess process) { + if (!verbose) { + return null; + } + // Add always, even when we don't use it in checks. + final StringBuilder report = new StringBuilder(4096); + if (checks.isPidFile()) { + report.append("\n The pid file ").append(pidFile.getAbsolutePath()); + if (pidFile.exists()) { + final Long pid = ProcessUtils.loadPid(pidFile); + if (pid == null) { + report.append(" exists but does not contain parseable pid."); + } else { + report.append(" contains pid ").append(pid).append('.'); + if (pid.longValue() != process.pid()) { + report.append(" WARNING: The process we started has different pid!"); + report.append(" The process with the pid ").append(pid); + report.append(ProcessUtils.isAlive(pid) ? " is" : " is not").append(" alive."); + } + } + } else { + report.append(" does not exist."); + } + } + report.append("\n Process with pid ").append(process.pid()).append(" is "); + report.append(process.isAlive() ? "alive" : "dead"); + + if (checks.isAdminEndpoint()) { + List adminEndpoints = adminEndpointsSupplier.get(); + if (!adminEndpoints.isEmpty()) { + report.append("\n Admin Endpoints:"); + appendEndpoints(adminEndpoints, report); + } + } + + if (!checks.getCustomEndpoints().isEmpty()) { + report.append("\n Custom Endpoints:"); + appendEndpoints(checks.getCustomEndpoints(), report); + } + return report.toString(); + } + + private void appendEndpoints(List endpoints, final StringBuilder report) { + for (HostAndPort endpoint : endpoints) { + final boolean listening = ProcessUtils.isListening(endpoint); + report.append("\n ").append(endpoint.getHost()).append(':').append(endpoint.getPort()); + report.append(' ').append(listening ? "is" : "is not").append(" reachable."); + } + } + + private String getSuggestions() { + return "Please see the server log files for command status.\n" + + "You can also start with the --verbose option in order to see early messages in this output."; + } + + /** + * Any - some endpoints might not be accessible from this host. + * + * @return true if found endpoint which seems working. + */ + private boolean isListeningOnAnyEndpoint(List endpoints) { + for (HostAndPort endpoint : endpoints) { + if (ProcessUtils.isListening(endpoint)) { + LOG.log(Level.TRACE, "Server is listening on {0}.", endpoint); + return true; + } + } + return false; + } + + /** + * All - user provided what to check. + * + * @return true if found endpoint which seems working. + */ + private boolean isListeningOnAllEndpoints(List endpoints) { + for (HostAndPort endpoint : endpoints) { + if (!ProcessUtils.isListening(endpoint)) { + LOG.log(Level.TRACE, "Server is not listening on {0}.", endpoint); + return false; + } + } + return true; + } + + public static Duration step(String message, Duration timeout, Action action) throws CommandException { + if (timeout != null && timeout.isNegative()) { + return timeout; + } + if (message != null) { + LOG.log(INFO, message); + } + Instant start = Instant.now(); + action.action(); + Duration stopDuration = Duration.between(start, Instant.now()); + return timeout == null ? null : timeout.minus(stopDuration); + } + + @FunctionalInterface + public interface Action { + void action() throws CommandException; + } + + + public static final class ServerLifeSigns { + private boolean error; + private String summary; + private String situationReport; + private String suggestion; + + public boolean isError() { + return error; + } + + public String getSummary() { + return summary; + } + + public String getSituationReport() { + return situationReport; + } + + public String getSuggestion() { + return suggestion; + } + } + + + public interface GlassFishProcess { + boolean isAlive(); + long pid(); + Integer exitCode(); + + static GlassFishProcess of(Process process) { + return new GlassFishProcessInstance(process); + } + static GlassFishProcess of(long pid) { + return new GlassFishProcessHandle(pid); + } + } + + + private static final class GlassFishProcessInstance implements GlassFishProcess { + + private final Process process; + + private GlassFishProcessInstance(Process process) { + this.process = process; + } + + @Override + public boolean isAlive() { + return process.isAlive(); + } + + @Override + public long pid() { + return process.pid(); + } + + @Override + public Integer exitCode() { + return process.exitValue(); + } + } + + private static final class GlassFishProcessHandle implements GlassFishProcess { + private final long pid; + + private GlassFishProcessHandle(long pid) { + this.pid = pid; + } + + @Override + public boolean isAlive() { + return ProcessUtils.isAlive(pid); + } + + @Override + public long pid() { + return pid; + } + + @Override + public Integer exitCode() { + return null; + } + } +} diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartDomainCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartDomainCommand.java index 6e458f7ebf2..bac0cd0ce07 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartDomainCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartDomainCommand.java @@ -25,8 +25,8 @@ import com.sun.enterprise.admin.util.CommandModelData.ParamModelData; import com.sun.enterprise.universal.process.ProcessStreamDrainer; import com.sun.enterprise.universal.xml.MiniXmlParserException; - -import jakarta.inject.Inject; +import com.sun.enterprise.util.HostAndPort; +import com.sun.enterprise.util.ObjectAnalyzer; import java.io.IOException; import java.time.Duration; @@ -34,6 +34,7 @@ import java.util.Arrays; import java.util.List; import java.util.Set; +import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -41,20 +42,15 @@ import org.glassfish.api.admin.CommandException; import org.glassfish.api.admin.CommandValidationException; import org.glassfish.api.admin.RuntimeType; -import org.glassfish.api.admin.ServerEnvironment; import org.glassfish.hk2.api.PerLookup; import org.glassfish.main.jdke.i18n.LocalStringsImpl; import org.glassfish.security.common.FileRealmHelper; import org.jvnet.hk2.annotations.Service; -import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_DEBUG_OFF; -import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_DEBUG_ON; -import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_NORMAL; +import static com.sun.enterprise.admin.cli.CLIConstants.MASTER_PASSWORD; import static com.sun.enterprise.admin.cli.CLIConstants.WAIT_FOR_DAS_TIME_MS; -import static com.sun.enterprise.admin.cli.CLIConstants.WALL_CLOCK_START_PROP; import static java.util.logging.Level.FINER; import static org.glassfish.api.admin.RuntimeType.DAS; -import static org.glassfish.main.jdke.props.SystemProperties.setProperty; /** * The start-domain command. @@ -66,7 +62,7 @@ @PerLookup public class StartDomainCommand extends LocalDomainCommand implements StartServerCommand { - private static final LocalStringsImpl strings = new LocalStringsImpl(StartDomainCommand.class); + private static final LocalStringsImpl I18N = new LocalStringsImpl(StartDomainCommand.class); @Param(optional = true, shortName = "v", defaultValue = "false") private boolean verbose; @@ -89,23 +85,35 @@ public class StartDomainCommand extends LocalDomainCommand implements StartServe @Param(optional = true, shortName = "s", defaultValue = "false") private boolean suspend; - @Param(name = "domain_name", primary = true, optional = true) - private String domainName0; - @Param(name = "dry-run", shortName = "n", optional = true, defaultValue = "false") - private boolean dry_run; + private boolean dryRun; @Param(optional = true) private Integer timeout; + @Param(name = "check-pid-file", optional = true, defaultValue = "true") + private boolean checkPidFile; + + @Param(name = "check-process-alive", optional = true, defaultValue = "true") + private boolean checkProcessAlive; + + @Param(name = "check-admin-port", optional = true, defaultValue = "true") + private boolean checkAdminEndpoint; + + @Param(name = "server-output", shortName = "o", optional = true) + private Boolean printServerOutput; + + @Param(name = "custom-endpoints", optional = true) + private String customEndpoints; + @Param(name = "drop-interrupted-commands", optional = true, defaultValue = "false") - private boolean drop_interrupted_commands; + private boolean dropInterruptedCommands; - @Inject - ServerEnvironment serverEnvironment; + @Param(name = "domain_name", primary = true, optional = true) + private String userArgDomainName; private GFLauncherInfo launchParameters; - private GFLauncher glassFishLauncher; + private GFLauncher launcher; private StartServerHelper startServerHelper; // the name of the master password option @@ -116,9 +124,6 @@ public RuntimeType getType() { return DAS; } - /** - * @return timeout for the command - */ @Override public Duration getTimeout() { return timeout == null ? WAIT_FOR_DAS_TIME_MS : Duration.ofSeconds(timeout); @@ -126,7 +131,7 @@ public Duration getTimeout() { @Override protected void validate() throws CommandException, CommandValidationException { - setDomainName(domainName0); + setDomainName(userArgDomainName); super.validate(); } @@ -134,33 +139,33 @@ protected void validate() throws CommandException, CommandValidationException { protected int executeCommand() throws CommandException { try { // createLauncher needs to go before the startServerHelper is created!! - createLauncher(); - String masterPassword = getMasterPassword(); + launcher = createLauncher(); - startServerHelper = new StartServerHelper(programOpts.isTerse(), getServerDirs(), glassFishLauncher, masterPassword, getTimeout()); + final List userEndpoints = StartServerHelper.parseCustomEndpoints(customEndpoints); + final ServerLifeSignCheck signOfLife = new ServerLifeSignCheck("domain " + getDomainName(), + printServerOutput, checkPidFile, checkProcessAlive, checkAdminEndpoint, userEndpoints); + startServerHelper = new StartServerHelper(programOpts.isTerse(), getTimeout(), getServerDirs(), launcher, signOfLife); - if (!startServerHelper.prepareForLaunch()) { + if (!upgrade && launcher.needsManualUpgrade()) { + logger.info(I18N.get("manualUpgradeNeeded")); return ERROR; } - if (!upgrade && glassFishLauncher.needsManualUpgrade()) { - logger.info(strings.get("manualUpgradeNeeded")); - return ERROR; + if (!upgrade && launcher.needsAutoUpgrade()) { + doAutoUpgrade(); } - doAutoUpgrade(masterPassword); - - if (dry_run) { + if (dryRun) { logger.fine("Dump of JVM Invocation line that would be used to launch:"); - List cmd = glassFishLauncher.getCommandLine().toList(); + List cmd = launcher.getCommandLine().toList(); int indexOfReadStdin = cmd.indexOf("-read-stdin"); String cmdToLog = IntStream.range(0, cmd.size()) - // Don't print -read-stdin option as it's not needed to run the server - // Also skip the next line with "true", which is related to this option - .filter(index -> index < indexOfReadStdin || index > indexOfReadStdin + 1) - .mapToObj(cmd::get) - .collect(Collectors.joining("\n")) - + "\n"; + // Don't print -read-stdin option as it's not needed to run the server + // Also skip the next line with "true", which is related to this option + .filter(index -> index < indexOfReadStdin || index > indexOfReadStdin + 1) + .mapToObj(cmd::get) + .collect(Collectors.joining("\n")) + + "\n"; logger.info(cmdToLog); return SUCCESS; } @@ -169,70 +174,45 @@ protected int executeCommand() throws CommandException { // Launch returns very quickly if verbose is not set // if verbose is set then it returns after the domain dies - glassFishLauncher.launch(); - - if (verbose || upgrade || watchdog) { // we can potentially loop forever here... - while (true) { - int returnValue = glassFishLauncher.getExitValue(); - - switch (returnValue) { - case RESTART_NORMAL: - logger.info(strings.get("restart")); - break; - case RESTART_DEBUG_ON: - logger.info(strings.get("restartChangeDebug", "on")); - launchParameters.setDebug(true); - break; - case RESTART_DEBUG_OFF: - logger.info(strings.get("restartChangeDebug", "off")); - launchParameters.setDebug(false); - break; - default: - return returnValue; - } - - if (env.debug()) { - setProperty(WALL_CLOCK_START_PROP, Long.toString(System.currentTimeMillis()), true); - } - - glassFishLauncher.setup(); - glassFishLauncher.launch(); - } - - } else { - startServerHelper.waitForServerStart(); - startServerHelper.report(); - return SUCCESS; + launcher.launch(); + + if (verbose || upgrade || watchdog) { + return startServerHelper.talkWithUser(); } - } catch (GFLauncherException gfle) { - throw new CommandException(gfle.getMessage()); + final String report = startServerHelper.waitForServerStart(getTimeout()); + logger.info(report); + return SUCCESS; + } catch (GFLauncherException e) { + throw new CommandException(e.getMessage(), e); } catch (MiniXmlParserException me) { throw new CommandException(me); } } - /** - * Create a glassFishLauncher for the domain specified by arguments to this command. The glassFishLauncher is for a - * server of the specified type. Sets the glassFishLauncher and launchParameters fields. It has to be public because it - * is part of an interface - */ @Override - public void createLauncher() throws GFLauncherException, MiniXmlParserException { - glassFishLauncher = GFLauncherFactory.getInstance(getType()); - launchParameters = glassFishLauncher.getInfo(); - + public final GFLauncher createLauncher() throws GFLauncherException, MiniXmlParserException, CommandException { + final GFLauncher gfLauncher = GFLauncherFactory.getInstance(getType()); + launchParameters = gfLauncher.getInfo(); launchParameters.setDomainName(getDomainName()); launchParameters.setDomainParentDir(getDomainsDir().getPath()); launchParameters.setVerbose(verbose || upgrade); + launchParameters.setIgnoreOutput(printServerOutput == Boolean.FALSE); launchParameters.setSuspend(suspend); launchParameters.setDebug(debug); launchParameters.setUpgrade(upgrade); launchParameters.setWatchdog(watchdog); - launchParameters.setDropInterruptedCommands(drop_interrupted_commands); - - launchParameters.setRespawnInfo(programOpts.getClassName(), programOpts.getModulePath(), programOpts.getClassPath(), respawnArgs()); + launchParameters.setDropInterruptedCommands(dropInterruptedCommands); + launchParameters.setRespawnInfo(programOpts.getClassName(), programOpts.getModulePath(), + programOpts.getClassPath(), respawnArgs()); + launchParameters.setAsadminAdminAddress(getUserProvidedAdminAddress()); + gfLauncher.setup(); + launchParameters.addSecurityToken(MASTER_PASSWORD, getMasterPassword()); + return gfLauncher; + } - glassFishLauncher.setup(); + @Override + public String toString() { + return ObjectAnalyzer.toStringWithSuper(this); } /** @@ -243,96 +223,101 @@ private String[] respawnArgs() { args.addAll(Arrays.asList(programOpts.getProgramArguments())); // now the start-domain specific arguments - args.add(getName()); // the command name + // the command name + args.add(getName()); args.add("--verbose=" + String.valueOf(verbose)); args.add("--watchdog=" + String.valueOf(watchdog)); args.add("--debug=" + String.valueOf(debug)); args.add("--domaindir"); args.add(getDomainsDir().toString()); if (ok(getDomainName())) { - args.add(getDomainName()); // the operand + // the operand + args.add(getDomainName()); } logger.log(FINER, "Respawn args: {0}", args); return args.toArray(String[]::new); } + /** - * If this domain needs to be upgraded and --upgrade wasn't specified, first start the domain to do the upgrade and then - * start the domain again for real. + * If this domain needs to be upgraded and --upgrade wasn't specified, first start the domain + * to do the upgrade and then start the domain again for real. + * + * @return new {@link GFLauncher} */ - private void doAutoUpgrade(String mpv) throws GFLauncherException, MiniXmlParserException, CommandException { - if (upgrade || !glassFishLauncher.needsAutoUpgrade()) { - return; + private GFLauncher doAutoUpgrade() throws GFLauncherException, MiniXmlParserException, CommandException { + logger.info(I18N.get("upgradeNeeded")); + launchParameters.setUpgrade(true); + launcher.setup(); + launcher.launch(); + final int exitCode = waitForAutoUpgradeFinish(); + if (exitCode == SUCCESS) { + logger.info(I18N.get("upgradeSuccessful")); + // need a new glassFishLauncher to start the domain for real + return createLauncher(); } + final ProcessStreamDrainer psd = launcher.getProcessStreamDrainer(); + final String output = psd.getOutErrString(); + if (ok(output)) { + throw new CommandException(I18N.get("upgradeFailedOutput", launchParameters.getDomainName(), exitCode, output)); + } + throw new CommandException(I18N.get("upgradeFailed", launchParameters.getDomainName(), exitCode)); + } - logger.info(strings.get("upgradeNeeded")); - launchParameters.setUpgrade(true); - glassFishLauncher.setup(); - glassFishLauncher.launch(); - Process glassFishProcess = glassFishLauncher.getProcess(); - int exitCode = -1; + private int waitForAutoUpgradeFinish() { + final Process glassFishProcess = launcher.getProcess(); try { - exitCode = glassFishProcess.waitFor(); - } catch (InterruptedException ex) { + return glassFishProcess.waitFor(); + } catch (InterruptedException e) { Thread.currentThread().interrupt(); + logger.log(Level.SEVERE, "Waiting for the upgrade was interrupted!", e); + System.exit(-1); + return -1; } - - if (exitCode != SUCCESS) { - ProcessStreamDrainer psd = glassFishLauncher.getProcessStreamDrainer(); - String output = psd.getOutErrString(); - if (ok(output)) { - throw new CommandException(strings.get("upgradeFailedOutput", launchParameters.getDomainName(), exitCode, output)); - } - throw new CommandException(strings.get("upgradeFailed", launchParameters.getDomainName(), exitCode)); - } - logger.info(strings.get("upgradeSuccessful")); - - // need a new glassFishLauncher to start the domain for real - createLauncher(); - // continue with normal start... } /** - * Check to make sure that at least one admin user is able to login. If none is found, then prompt for an admin - * password. + * Check to make sure that at least one admin user is able to login. + * If none is found, then prompt for an admin password. * * NOTE: this depends on glassFishLauncher.setup having already been called. */ private void doAdminPasswordCheck() throws CommandException { - String adminRealmKeyFile = glassFishLauncher.getAdminRealmKeyFile(); - if (adminRealmKeyFile != null) { - try { - FileRealmHelper fileRealmHelper = new FileRealmHelper(adminRealmKeyFile); - if (!fileRealmHelper.hasAuthenticatableUser()) { - - // Prompt for the password for the first user and set it - Set adminUsers = fileRealmHelper.getUserNames(); - if (adminUsers == null || adminUsers.isEmpty()) { - throw new CommandException("no admin users"); - } - - String firstAdminUser = adminUsers.iterator().next(); - ParamModelData npwo = new ParamModelData(newpwName, String.class, false, null); - npwo.prompt = strings.get("new.adminpw", firstAdminUser); - npwo.promptAgain = strings.get("new.adminpw.again", firstAdminUser); - npwo.param._password = true; - - logger.info(strings.get("new.adminpw.prompt")); - char[] newPasswordArray = super.getPassword(npwo, null, true); - String newPassword = newPasswordArray != null ? new String(newPasswordArray) : null; - if (newPassword == null) { - throw new CommandException("The Master Password is required to start the domain.\n" - + "No console, no prompting possible. You should either create the domain\n" - + "with --savemasterpassword=true or provide a password file with the --passwordfile option."); - } - - fileRealmHelper.updateUser(firstAdminUser, firstAdminUser, newPassword.toCharArray(), null); - fileRealmHelper.persist(); - } - } catch (IOException ioe) { - throw new CommandException(ioe); + String adminRealmKeyFile = launcher.getAdminRealmKeyFile(); + if (adminRealmKeyFile == null) { + return; + } + try { + FileRealmHelper fileRealmHelper = new FileRealmHelper(adminRealmKeyFile); + if (fileRealmHelper.hasAuthenticatableUser()) { + return; } + // Prompt for the password for the first user and set it + Set adminUsers = fileRealmHelper.getUserNames(); + if (adminUsers == null || adminUsers.isEmpty()) { + throw new CommandException("no admin users"); + } + + String firstAdminUser = adminUsers.iterator().next(); + ParamModelData npwo = new ParamModelData(newpwName, String.class, false, null); + npwo.prompt = I18N.get("new.adminpw", firstAdminUser); + npwo.promptAgain = I18N.get("new.adminpw.again", firstAdminUser); + npwo.param._password = true; + + logger.info(I18N.get("new.adminpw.prompt")); + char[] newPasswordArray = super.getPassword(npwo, null, true); + String newPassword = newPasswordArray == null ? null : new String(newPasswordArray); + if (newPassword == null) { + throw new CommandException("The Master Password is required to start the domain.\n" + + "No console, no prompting possible. You should either create the domain\n" + + "with --savemasterpassword=true or provide a password file with the --passwordfile option."); + } + + fileRealmHelper.updateUser(firstAdminUser, firstAdminUser, newPassword.toCharArray(), null); + fileRealmHelper.persist(); + } catch (IOException ioe) { + throw new CommandException(ioe); } } } diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerCommand.java index cbaae7567fa..5aeebf76676 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerCommand.java @@ -16,11 +16,13 @@ */ package com.sun.enterprise.admin.servermgmt.cli; +import com.sun.enterprise.admin.launcher.GFLauncher; import com.sun.enterprise.admin.launcher.GFLauncherException; import com.sun.enterprise.universal.xml.MiniXmlParserException; import java.time.Duration; +import org.glassfish.api.admin.CommandException; import org.glassfish.api.admin.RuntimeType; /** @@ -34,9 +36,12 @@ public interface StartServerCommand { RuntimeType getType(); /** - * Create a launcher for the whatever type of server "we" are. + * @return new {@link GFLauncher} for the whatever type of server "we" are. + * @throws GFLauncherException Failed to create the launcher for some reason + * @throws MiniXmlParserException Failed to load domain.xml + * @throws CommandException */ - void createLauncher() throws GFLauncherException, MiniXmlParserException; + GFLauncher createLauncher() throws GFLauncherException, MiniXmlParserException, CommandException; /** * Timeout for the individual command. diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java index 2d308c7b407..9817e63d1c6 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java @@ -18,9 +18,12 @@ package com.sun.enterprise.admin.servermgmt.cli; import com.sun.enterprise.admin.launcher.GFLauncher; +import com.sun.enterprise.admin.launcher.GFLauncherException; import com.sun.enterprise.admin.launcher.GFLauncherInfo; -import com.sun.enterprise.universal.process.ProcessStreamDrainer; +import com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignChecker.GlassFishProcess; +import com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignChecker.ServerLifeSigns; import com.sun.enterprise.universal.process.ProcessUtils; +import com.sun.enterprise.universal.xml.MiniXmlParserException; import com.sun.enterprise.util.HostAndPort; import com.sun.enterprise.util.io.ServerDirs; import com.sun.enterprise.util.net.NetUtils; @@ -28,32 +31,39 @@ import java.io.File; import java.io.IOException; import java.lang.System.Logger; -import java.lang.System.Logger.Level; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.ArrayList; import java.util.List; -import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.stream.Collectors; import org.glassfish.api.admin.CommandException; import org.glassfish.main.jdke.i18n.LocalStringsImpl; +import org.glassfish.main.jul.formatter.OneLineFormatter; +import org.glassfish.main.jul.handler.GlassFishLogHandler; +import org.glassfish.main.jul.handler.GlassFishLogHandlerConfiguration; -import static com.sun.enterprise.admin.cli.CLIConstants.MASTER_PASSWORD; +import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_DEBUG_OFF; +import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_DEBUG_ON; +import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_NORMAL; +import static com.sun.enterprise.admin.cli.CLIConstants.WALL_CLOCK_START_PROP; +import static com.sun.enterprise.universal.process.ProcessUtils.waitWhileListening; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.INFO; -import static java.lang.System.Logger.Level.WARNING; +import static org.glassfish.main.jdke.props.SystemProperties.setProperty; /** - * Java does not allow multiple inheritance. Both StartDomainCommand and StartInstanceCommand have common code but they - * are already in a different hierarchy of classes. The first common baseclass is too far away -- e.g. no "launcher" - * variable, etc. - * - * Instead -- put common code in here and call it as common utilities This class is designed to be thread-safe and - * IMMUTABLE + * Manager of starting process. + * While {@link GFLauncher} manages the launch of the new server, this controls also user interaction. * * @author bnevins + * @author David Matejcek */ -public class StartServerHelper { +public final class StartServerHelper { private static final LocalStringsImpl I18N = new LocalStringsImpl(StartServerHelper.class); private static final Logger LOG = System.getLogger(StartServerHelper.class.getName(), I18N.getBundle()); @@ -61,170 +71,144 @@ public class StartServerHelper { private final GFLauncher launcher; private final File pidFile; private final GFLauncherInfo info; - private final List addresses; + private final List adminAddresses; private final ServerDirs serverDirs; - private final String masterPassword; - private final String serverOrDomainName; - private final Duration timeout; + private final String serverTitleAndName; + private final ServerLifeSignCheck lifeSignCheck; - public StartServerHelper(boolean terse, ServerDirs serverDirs, GFLauncher launcher, String masterPassword, - Duration timeout) { + public StartServerHelper(boolean terse, Duration timeout, ServerDirs serverDirs, GFLauncher launcher, + ServerLifeSignCheck lifeSignCheck) throws GFLauncherException { this.terse = terse; this.launcher = launcher; this.info = launcher.getInfo(); - - if (info.isDomain()) { - this.serverOrDomainName = info.getDomainName(); - } else { - this.serverOrDomainName = info.getInstanceName(); - } - - this.addresses = info.getAdminAddresses(); + this.serverTitleAndName = (info.isDomain() ? "domain " : "instance ") + serverDirs.getServerName(); + this.adminAddresses = info.getAdminAddresses(); this.serverDirs = serverDirs; this.pidFile = serverDirs.getPidFile(); - this.masterPassword = masterPassword; - this.timeout = timeout; - } - - - public void waitForServerStart() throws CommandException { - if (!terse) { - // use stdout because logger always appends a newline - System.out.print(I18N.get("WaitServer", serverOrDomainName) + " "); - } + this.lifeSignCheck = lifeSignCheck; - final Process glassFishProcess = launcher.getProcess(); - final Supplier signOfFinishedStartup = () -> { - if (pidFile == null) { - if (isListeningOnAnyEndpoint()) { - return true; - } - } else { - if (pidFile.exists()) { - LOG.log(Level.TRACE, "The pid file {0} has been created.", pidFile); - return true; - } + // This means we are running restart. + // Restart has a problem, the start is initiate d by the running server. + // That means we cannot watch its output. So we have to write it down. + if (launcher.getPidBeforeRestart() != null) { + waitForParentToDie(launcher.getPidBeforeRestart(), timeout); + configureLoggingOfRestart(serverDirs.getRestartLogFile()); + final Integer debugPort = launcher.getDebugPort(); + if (debugPort != null) { + LOG.log(INFO, "Waiting few seconds until debug port {0} is free.", debugPort); + waitWhileListening(new HostAndPort("localhost", debugPort, true), Duration.ofSeconds(10), terse); } - // Don't wait if the process died. - return !glassFishProcess.isAlive(); - }; - if (!ProcessUtils.waitFor(signOfFinishedStartup, timeout, !terse)) { - final String msg; - if (info.isDomain()) { - msg = I18N.get("serverNoStart", I18N.get("DAS"), info.getDomainName(), timeout); - } else { - msg = I18N.get("serverNoStart", I18N.get("INSTANCE"), info.getInstanceName(), timeout); - } - throw new CommandException(msg); - } - - if (glassFishProcess.isAlive()) { - // Ok, server is running. - return; } - - // Now try to throw some comprehensible report about what happened. - final int exitCode = glassFishProcess.exitValue(); - ProcessStreamDrainer psd = launcher.getProcessStreamDrainer(); - final String output = psd.getOutErrString(); - final String serverName = info.isDomain() - ? "domain " + info.getDomainName() - : "instance " + info.getInstanceName(); - if (exitCode == 0) { - LOG.log(INFO, - "Server {0} started successfuly. The startup command produced following output before it finished: \n{1}", - serverName, output); - return; - } - if (output.isEmpty()) { - throw new CommandException(I18N.get("serverDied", serverName, exitCode)); - } - throw new CommandException(I18N.get("serverDiedOutput", serverName, exitCode, output)); + checkFreeAdminPorts(info.getAdminAddresses()); + deletePidFile(); } /** - * Run a series of commands to prepare for a launch. + * Blocks and communicates with the user using console. * - * @return false if there was a problem. + * @return launcher exit code + * @throws GFLauncherException + * @throws MiniXmlParserException */ - public boolean prepareForLaunch() throws CommandException { - waitForParentToDie(); - setSecurity(); - if (!checkPorts()) { - return false; + public int talkWithUser() throws GFLauncherException, MiniXmlParserException { + while (true) { + int returnValue = launcher.getExitValue(); + switch (returnValue) { + case RESTART_NORMAL: + LOG.log(INFO, "restart"); + break; + case RESTART_DEBUG_ON: + LOG.log(INFO, "restartChangeDebug", "on"); + info.setDebug(true); + break; + case RESTART_DEBUG_OFF: + LOG.log(INFO, "restartChangeDebug", "off"); + info.setDebug(false); + break; + default: + return returnValue; + } + setProperty(WALL_CLOCK_START_PROP, Instant.now().toString(), true); + launcher.setup(); + launcher.launch(); } - deletePidFile(); - return true; } - - public void report() { - final String logfile = launcher.getLogFilename(); - final Integer adminPort; - if (addresses == null || addresses.isEmpty()) { - adminPort = null; - } else { - adminPort = addresses.get(0).getPort(); + public String waitForServerStart(Duration timeout) throws GFLauncherException { + if (!terse) { + System.out.print("Waiting for " + serverTitleAndName + " to start "); + } + final GlassFishProcess glassFishProcess = GlassFishProcess.of(launcher.getProcess()); + final ServerLifeSignChecker checker = new ServerLifeSignChecker(lifeSignCheck, pidFile, () -> adminAddresses, !terse); + final ServerLifeSigns signs = checker.watchStartup(glassFishProcess, timeout); + final String report = report(signs); + if (signs.isError()) { + throw new GFLauncherException(report); } - LOG.log(INFO, "ServerStart.SuccessMessage", info.isDomain() ? "domain " : "instance", - serverDirs.getServerName(), serverDirs.getServerDir(), logfile, adminPort); + return report; } - /** - * If the parent is a GF server -- then wait for it to die. - * This is part of the Client-Server Restart Dance! - * The dying server called us with the system property AS_RESTART_PREVIOUS_PID set to its pid - * - * @throws CommandException if we timeout waiting for the parent to die or if the admin ports never free up - */ - private void waitForParentToDie() throws CommandException { - // we also come here with just a regular start in which case there is - // no parent, and the System Property is NOT set to anything... - final Integer pid = getParentPid(); - if (pid == null) { - return; + private String report(ServerLifeSigns signs) { + final StringBuilder report = new StringBuilder(2048); + report.append('\n').append(signs.getSummary()); + if (signs.getSuggestion() != null) { + report.append('\n').append(signs.getSuggestion()); } - LOG.log(DEBUG, "Waiting for death of the parent process with pid={0}", pid); - if (!ProcessUtils.waitWhileIsAlive(pid, timeout, false)) { - throw new CommandException(I18N.get("deathwait_timeout", timeout)); + report.append("\n Location: ").append(serverDirs.getServerDir()); + report.append("\n Log File: ").append(launcher.getLogFilename()); + final String situationReport = signs.getSituationReport(); + if (situationReport != null) { + report.append(signs.getSituationReport()); + } + // Print output just if user explicitly asked + // or start failed and user did not explicitly forbid the print. + if (launcher.getPidBeforeRestart() != null + || lifeSignCheck.getPrintServerOutput() == Boolean.TRUE + || (signs.isError() && lifeSignCheck.getPrintServerOutput() != Boolean.FALSE)) { + report.append("\n\n").append(getProcessOutput()); } - LOG.log(DEBUG, "Parent process with PID={0} is dead and all admin endpoints are free.", pid); + return report.append('\n').toString(); } - private Integer getParentPid() { - String pid = System.getProperty("AS_RESTART_PREVIOUS_PID"); - if (pid == null) { - return null; - } - try { - return Integer.valueOf(pid); - } catch (NumberFormatException e) { - LOG.log(WARNING, "Cannot parse pid {0} required for waiting for the death of the parent process.", pid); - return null; + private String getProcessOutput() { + final String output = launcher.getProcessStreamDrainer().getOutErrString(); + if (output == null) { + return "Unfortunately the new process did not produce any output."; } + return "The output of the process until we stopped watching:\n\n**********\n" + output + "\n**********"; } - - private boolean isListeningOnAnyEndpoint() { - for (HostAndPort address : addresses) { - if (ProcessUtils.isListening(address)) { - LOG.log(Level.TRACE, "Server is listening on {0}.", address); - return true; - } + private void configureLoggingOfRestart(File logFile) { + GlassFishLogHandlerConfiguration cfg = new GlassFishLogHandlerConfiguration(); + cfg.setFormatterConfiguration(new OneLineFormatter()); + if (logFile.isFile()) { + logFile.renameTo(new File(logFile.getParent(), "restart.log_" + LocalDateTime.now())); } - return false; + cfg.setLogFile(logFile); + cfg.setLevel(Level.ALL); + cfg.setFlushFrequency(1); + GlassFishLogHandler handler = new GlassFishLogHandler(cfg); + java.util.logging.Logger.getLogger("").addHandler(handler); + // see AdminMain + java.util.logging.Logger.getLogger("com.sun.enterprise.admin.cli").addHandler(handler); } - - private boolean checkPorts() { - String err = adminPortInUse(); - if (err == null) { - return true; + /** + * If the parent is a GF server -- then wait for it to die. + * This is part of the Client-Server Restart Dance! + * The dying server called us with the system property AS_RESTART_PREVIOUS_PID set to its pid + * + * @throws CommandException if we timeout waiting for the parent to die or if the admin ports never free up + */ + private void waitForParentToDie(long pid, Duration timeout) throws GFLauncherException { + LOG.log(INFO, () -> "Waiting for death of the parent process with the pid " + pid); + if (!ProcessUtils.waitWhileIsAlive(pid, timeout, false)) { + throw new GFLauncherException("Waited " + timeout.toSeconds() + + " s for the server to die. Restart is not possible unless you kill it manually."); } - LOG.log(WARNING, err); - return false; + LOG.log(DEBUG, () -> "Parent process with PID " + pid + " is dead."); } private void deletePidFile() { @@ -241,22 +225,65 @@ private void deletePidFile() { LOG.log(DEBUG, "The pid file {0} has been deleted.", pidFile); } - private void setSecurity() { - info.addSecurityToken(MASTER_PASSWORD, masterPassword); - } - private String adminPortInUse() { - return adminPortInUse(info.getAdminAddresses()); + /** + * @param endpoints + * @return space separated list of endpoints including HTTP/HTTPS protocol prefix. + */ + public static String toHttpList(List endpoints) { + return endpoints.stream() + .map(h -> (h.isSecure() ? "https://" : "http://") + h.getHost() + ':' + h.getPort()) + .collect(Collectors.joining(" ")); } - private static String adminPortInUse(List adminAddresses) { - // it returns a String for logging --- if desired - for (HostAndPort addr : adminAddresses) { - if (!NetUtils.isPortFree(addr.getHost(), addr.getPort())) { - return I18N.get("ServerRunning", addr.getPort()); + + /** + * @param customEndpoints value provided by the user + * @return parsed list of endpoints to check + * @throws CommandException if anything goes wrong + */ + public static List parseCustomEndpoints(String customEndpoints) throws CommandException { + if (customEndpoints == null || customEndpoints.isBlank()) { + return List.of(); + } + final String[] strings = customEndpoints.strip().split(","); + final List endpoints = new ArrayList<>(strings.length); + for (String string : strings) { + try { + final boolean secure = string.startsWith("https"); + final boolean http = string.startsWith("http"); + final String[] pair = string.replaceFirst("[a-z]+\\://", "").split(":"); + final HostAndPort endpoint; + if (pair.length == 1) { + final int port; + if (http) { + port = secure ? 443 : 80; + } else { + throw new CommandException("Port is mandatory endpoints without explicit protocol: " + string); + } + endpoint = new HostAndPort(pair[0], port, secure); + } else if (pair.length == 2) { + endpoint = new HostAndPort(pair[0], Integer.parseInt(pair[1]), secure); + } else { + throw new CommandException("Invalid customEndpoints value: " + string); + } + endpoints.add(endpoint); + } catch (CommandException e) { + throw e; + } catch (Exception e) { + throw new CommandException("Invalid customEndpoints value: " + string, e); } } + return endpoints; + } - return null; + private static void checkFreeAdminPorts(List endpoints) throws GFLauncherException { + LOG.log(INFO, "Checking if all admin ports are free."); + for (HostAndPort endpoint : endpoints) { + if (!NetUtils.isPortFree(endpoint.getHost(), endpoint.getPort())) { + throw new GFLauncherException("There is a process already using the admin port " + endpoint.getPort() + + " - it might be another instance of a GlassFish server."); + } + } } } diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StopDomainCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StopDomainCommand.java index 8fcd75fc4e3..883ef2c7c03 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StopDomainCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StopDomainCommand.java @@ -106,7 +106,7 @@ protected int executeCommand() throws CommandException { return dasNotRunning(); } programOpts.setHostAndPort(addr); - LOG.log(Level.DEBUG, "Stopping local domain on port {0}", programOpts.getPort()); + LOG.log(Level.INFO, "Stopping local domain on admin endpoint {0}", addr); /* * If we're using the local password, we don't want to prompt @@ -119,15 +119,14 @@ protected int executeCommand() throws CommandException { if (!isThisDAS(getDomainRootDir())) { return dasNotRunning(); } - LOG.log(Level.DEBUG, "It's the correct DAS"); } else { + addr = getUserProvidedAdminAddress(); // remote // Verify that the DAS is running and reachable if (!DASUtils.pingDASQuietly(programOpts, env)) { return dasNotRunning(); } - LOG.log(Level.DEBUG, "DAS is running"); programOpts.setInteractive(false); } @@ -178,27 +177,18 @@ protected int dasNotRunning() throws CommandException { */ protected void doCommand() throws CommandException { // run the remote stop-domain command and throw away the output - final RemoteCLICommand cmd = new RemoteCLICommand(getName(), programOpts, env); - final Long oldPid = getServerPid(); + final Long pid = getServerPid(); final boolean printDots = !programOpts.isTerse(); final Duration stopTimeout = getStopTimeout(); + localShutdown(isLocal() ? pid : null, stopTimeout, printDots); + } + + private void localShutdown(Long pid, Duration stopTimeout, boolean printDots) throws CommandException { + final RemoteCLICommand cmd = new RemoteCLICommand(getName(), programOpts, env); + cmd.executeAndReturnOutput("stop-domain", "--force", force.toString()); try { - cmd.executeAndReturnOutput("stop-domain", "--force", force.toString()); - if (printDots) { - // use stdout because logger always appends a newline - System.out.print("Waiting for the domain to stop "); - } - final boolean dead; - if (isLocal()) { - dead = oldPid == null || ProcessUtils.waitWhileIsAlive(oldPid, stopTimeout, printDots); - } else { - dead = ProcessUtils.waitWhileListening(addr, stopTimeout, printDots); - } - if (!dead) { - throw new CommandException( - "Timed out " + stopTimeout.toSeconds() + " seconds waiting for the domain to stop."); - } - } catch (Exception e) { + waitForStop(pid, addr, stopTimeout); + } catch (CommandException e) { // The domain server may have died so fast we didn't have time to // get the (always successful!!) return data. This is NOT AN ERROR! LOG.log(Level.DEBUG, "Remote stop-domain call failed.", e); diff --git a/nucleus/admin/server-mgmt/src/test/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelperTest.java b/nucleus/admin/server-mgmt/src/test/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelperTest.java new file mode 100644 index 00000000000..5d54af445ed --- /dev/null +++ b/nucleus/admin/server-mgmt/src/test/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelperTest.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2025 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0, which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the + * Eclipse Public License v. 2.0 are satisfied: GNU General Public License, + * version 2 with the GNU Classpath Exception, which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + */ + +package com.sun.enterprise.admin.servermgmt.cli; + +import com.sun.enterprise.util.HostAndPort; + +import org.glassfish.api.admin.CommandException; +import org.junit.jupiter.api.Test; + +import static com.sun.enterprise.admin.servermgmt.cli.StartServerHelper.parseCustomEndpoints; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class StartServerHelperTest { + + @Test + void parseEndpoints() throws Exception { + assertThat(parseCustomEndpoints(null), empty()); + assertThat(parseCustomEndpoints(" "), empty()); + assertThat(parseCustomEndpoints("carrot:123,https://google.com,sftp://xy:21"), + contains( + new HostAndPort("carrot", 123, false), + new HostAndPort("google.com", 443, true), + new HostAndPort("xy", 21, false) + )); + assertThrows(CommandException.class, () -> parseCustomEndpoints("sftp://x")); + } +} diff --git a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/CreateLocalInstanceCommand.java b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/CreateLocalInstanceCommand.java index c6aa6c3743b..b7a1734e91d 100644 --- a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/CreateLocalInstanceCommand.java +++ b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/CreateLocalInstanceCommand.java @@ -159,9 +159,8 @@ protected int executeCommand() if (node == null) { if(nodeDirChild == null) { - throw new CommandException(Strings.get("internal.error", - "nodeDirChild was null. The Base Class is supposed to " - + "guarantee that this won't happen")); + throw new CommandException( + "The nodeDirChild was null. The Base Class is supposed to guarantee that this won't happen"); } _node = nodeDirChild.getName(); String nodeHost = getInstanceHostName(true); diff --git a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/LocalStrings.properties b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/LocalStrings.properties index 5fd51828b44..2f67e0a8624 100644 --- a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/LocalStrings.properties +++ b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/LocalStrings.properties @@ -185,4 +185,3 @@ Please make sure you have the interactive flag set to true. See '--help' for de The other option is to use the --force option vld.noconsole=This command can only be run from a console. Please try again with a console attached. vld.no=You chose to not run the command. -internal.error=Internal Error: {0} diff --git a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/RestartLocalInstanceCommand.java b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/RestartLocalInstanceCommand.java index 19a93f3bd0d..c62444e457b 100644 --- a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/RestartLocalInstanceCommand.java +++ b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/RestartLocalInstanceCommand.java @@ -19,6 +19,7 @@ import com.sun.enterprise.admin.cli.CLICommand; import com.sun.enterprise.admin.cli.remote.RemoteCLICommand; +import com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignCheck; import com.sun.enterprise.util.HostAndPort; import jakarta.inject.Inject; @@ -35,6 +36,7 @@ import static com.sun.enterprise.admin.cli.CLIConstants.DEATH_TIMEOUT_MS; import static com.sun.enterprise.admin.cli.CLIConstants.WAIT_FOR_DAS_TIME_MS; +import static com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignChecker.step; /** * @author Byron Nevins @@ -61,7 +63,6 @@ protected final void doCommand() throws CommandException { // Save old values before executing restart final Long oldPid = getServerPid(); final HostAndPort oldAdminAddress = getReachableAdminAddress(); - final HostAndPort newAdminAddress = getAdminAddress(getServerDirs().getServerName()); // run the remote restart-instance command and throw away the output RemoteCLICommand cmd = new RemoteCLICommand("_restart-instance", programOpts, env); @@ -71,9 +72,13 @@ protected final void doCommand() throws CommandException { cmd.executeAndReturnOutput("_restart-instance", "--debug", debug.toString()); } - // Timeouts are set in commands we use, so we will wait for the result without timeout. - waitForRestart(oldPid, oldAdminAddress, newAdminAddress, getRestartTimeout()); - logger.info("Successfully restarted the instance."); + final Duration timeout = getRestartTimeout(); + final Duration startTimeout = step("Waiting until instance stops.", timeout, + () -> waitForStop(oldPid, oldAdminAddress, timeout)); + + final ServerLifeSignCheck lifeSignCheck = new ServerLifeSignCheck("instance " + getInstanceName(), true, true, true, true, List.of()); + final String report = waitForStart(oldPid, lifeSignCheck, () -> List.of(getReachableAdminAddress()), startTimeout); + logger.info(report); } @Override diff --git a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/StartLocalInstanceCommand.java b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/StartLocalInstanceCommand.java index c6248f70de0..0ac8adb2259 100644 --- a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/StartLocalInstanceCommand.java +++ b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/StartLocalInstanceCommand.java @@ -17,14 +17,15 @@ package com.sun.enterprise.admin.cli.cluster; -import com.sun.enterprise.admin.cli.CLIConstants; import com.sun.enterprise.admin.launcher.GFLauncher; import com.sun.enterprise.admin.launcher.GFLauncherException; import com.sun.enterprise.admin.launcher.GFLauncherFactory; import com.sun.enterprise.admin.launcher.GFLauncherInfo; +import com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignCheck; import com.sun.enterprise.admin.servermgmt.cli.StartServerCommand; import com.sun.enterprise.admin.servermgmt.cli.StartServerHelper; import com.sun.enterprise.universal.xml.MiniXmlParserException; +import com.sun.enterprise.util.HostAndPort; import com.sun.enterprise.util.ObjectAnalyzer; import java.io.File; @@ -41,11 +42,8 @@ import org.glassfish.hk2.api.PerLookup; import org.jvnet.hk2.annotations.Service; -import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_DEBUG_OFF; -import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_DEBUG_ON; -import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_NORMAL; +import static com.sun.enterprise.admin.cli.CLIConstants.MASTER_PASSWORD; import static com.sun.enterprise.admin.cli.CLIConstants.WAIT_FOR_DAS_TIME_MS; -import static org.glassfish.main.jdke.props.SystemProperties.setProperty; /** * Start a local server instance. @@ -65,15 +63,29 @@ public class StartLocalInstanceCommand extends SynchronizeInstanceCommand implem private boolean debug; @Param(name = "dry-run", shortName = "n", optional = true, defaultValue = "false") - private boolean dry_run; + private boolean dryRun; @Param(optional = true) private Integer timeout; - private GFLauncherInfo info; - private GFLauncher launcher; + @Param(name = "check-pid-file", optional = true, defaultValue = "true") + private boolean checkPidFile; + + @Param(name = "check-process-alive", optional = true, defaultValue = "true") + private boolean checkProcessAlive; + + @Param(name = "check-admin-port", optional = true, defaultValue = "true") + private boolean checkAdminEndpoint; - private StartServerHelper helper; + @Param(name = "server-output", shortName = "o", optional = true) + private Boolean printServerOutput; + + @Param(name = "custom-endpoints", optional = true) + private String customEndpoints; + + private GFLauncherInfo launchParameters; + private GFLauncher launcher; + private StartServerHelper startServerHelper; @Override public RuntimeType getType() { @@ -87,15 +99,11 @@ protected boolean mkdirs(File f) { return false; } - /** - * @return timeout for the command - */ @Override public Duration getTimeout() { return timeout == null ? WAIT_FOR_DAS_TIME_MS : Duration.ofSeconds(timeout); } - @Override protected void validate() throws CommandException { super.validate(); @@ -107,8 +115,6 @@ protected void validate() throws CommandException { } } - /** - */ @Override protected int executeCommand() throws CommandException { logger.finer(() -> toString()); @@ -127,85 +133,60 @@ protected int executeCommand() throws CommandException { } try { - // createLauncher needs to go before the helper is created!! - createLauncher(); - final String mpv = getMasterPassword(); + // createLauncher needs to go before the startServerHelper is created!! + launcher = createLauncher(); - helper = new StartServerHelper(programOpts.isTerse(), getServerDirs(), launcher, mpv, getTimeout()); + final List userEndpoints = StartServerHelper.parseCustomEndpoints(customEndpoints); + final ServerLifeSignCheck signOfLife = new ServerLifeSignCheck("instance " + getInstanceName(), + printServerOutput, checkPidFile, checkProcessAlive, checkAdminEndpoint, userEndpoints); + startServerHelper = new StartServerHelper(programOpts.isTerse(), getTimeout(), getServerDirs(), launcher, signOfLife); - if (!helper.prepareForLaunch()) { - return ERROR; - } - - if (dry_run) { + if (dryRun) { logger.log(Level.FINE, Strings.get("dry_run_msg")); - logger.log(Level.INFO, getLauncher().getCommandLine().toString("\n")); - return SUCCESS; - } - - getLauncher().launch(); - - if (!verbose && !watchdog) { - helper.waitForServerStart(); - helper.report(); + logger.log(Level.INFO, launcher.getCommandLine().toString("\n")); return SUCCESS; } - // we can potentially loop forever here... - while (true) { - int returnValue = getLauncher().getExitValue(); - switch (returnValue) { - case RESTART_NORMAL: - logger.info(Strings.get("restart")); - break; - case RESTART_DEBUG_ON: - logger.info(Strings.get("restartChangeDebug", "on")); - getInfo().setDebug(true); - break; - case RESTART_DEBUG_OFF: - logger.info(Strings.get("restartChangeDebug", "off")); - getInfo().setDebug(false); - break; - default: - return returnValue; - } + launcher.launch(); - if (env.debug()) { - setProperty(CLIConstants.WALL_CLOCK_START_PROP, Long.toString(System.currentTimeMillis()), true); - } - getLauncher().setup(); - getLauncher().launch(); + if (verbose || watchdog) { + return startServerHelper.talkWithUser(); } - } catch (GFLauncherException gfle) { - throw new CommandException(gfle.getMessage()); - } catch (MiniXmlParserException me) { - throw new CommandException(me); + final String report = startServerHelper.waitForServerStart(getTimeout()); + logger.info(report); + return SUCCESS; + } catch (GFLauncherException e) { + throw new CommandException(e.getMessage(), e); + } catch (MiniXmlParserException e) { + throw new CommandException(e.getMessage(), e); } } - /** - * Create a launcher for the instance specified by arguments to - * this command. The launcher is for a server of the specified type. - * Sets the launcher and info fields. - */ @Override - public void createLauncher() throws GFLauncherException, MiniXmlParserException { - setLauncher(GFLauncherFactory.getInstance(getType())); - setInfo(getLauncher().getInfo()); - getInfo().setInstanceName(instanceName); - getInfo().setInstanceRootDir(instanceDir); - getInfo().setVerbose(verbose); - getInfo().setWatchdog(watchdog); - getInfo().setDebug(debug); - getInfo().setRespawnInfo(programOpts.getClassName(), programOpts.getModulePath(), programOpts.getClassPath(), - respawnArgs()); - - getLauncher().setup(); + public final GFLauncher createLauncher() throws GFLauncherException, MiniXmlParserException, CommandException { + final GFLauncher gfLauncher = GFLauncherFactory.getInstance(getType()); + this.launchParameters = gfLauncher.getInfo(); + launchParameters.setInstanceName(instanceName); + launchParameters.setInstanceRootDir(instanceDir); + launchParameters.setVerbose(verbose); + launchParameters.setIgnoreOutput(printServerOutput == Boolean.FALSE); + launchParameters.setWatchdog(watchdog); + launchParameters.setDebug(debug); + launchParameters.setRespawnInfo(programOpts.getClassName(), programOpts.getModulePath(), + programOpts.getClassPath(), respawnArgs()); + launchParameters.setAsadminAdminAddress(getUserProvidedAdminAddress()); + gfLauncher.setup(); + launchParameters.addSecurityToken(MASTER_PASSWORD, getMasterPassword()); + return gfLauncher; + } + + @Override + public String toString() { + return ObjectAnalyzer.toStringWithSuper(this); } /** - * Return the asadmin command line arguments necessary to - * start this server instance. + * @return the asadmin command line arguments necessary to start this server instance. */ private String[] respawnArgs() { List args = new ArrayList<>(15); @@ -238,32 +219,4 @@ private String[] respawnArgs() { logger.log(Level.FINER, "Respawn args: {0}", args); return args.toArray(String[]::new); } - - private GFLauncher getLauncher() { - if(launcher == null) { - throw new RuntimeException(Strings.get("internal.error", "GFLauncher was not initialized")); - } - - return launcher; - } - private void setLauncher(GFLauncher gfl) { - launcher = gfl; - } - - private GFLauncherInfo getInfo() { - if (info == null) { - throw new RuntimeException(Strings.get("internal.error", "GFLauncherInfo was not initialized")); - } - return info; - } - - - private void setInfo(GFLauncherInfo inf) { - info = inf; - } - - @Override - public String toString() { - return ObjectAnalyzer.toStringWithSuper(this); - } } diff --git a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/StopLocalInstanceCommand.java b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/StopLocalInstanceCommand.java index 1be1117d1a3..92f06b29858 100644 --- a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/StopLocalInstanceCommand.java +++ b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/StopLocalInstanceCommand.java @@ -75,16 +75,6 @@ protected Duration getTimeout() { return timeout == null ? null : Duration.ofSeconds(timeout); } - /** - * Big trouble if you allow the super implementation to run - * because it creates directories. If this command is called with - * an instance that doesn't exist -- new dirs will be created which - * can cause other problems. - */ - @Override - protected void initInstance() throws CommandException { - super.initInstance(); - } @Override protected int executeCommand() throws CommandException, CommandValidationException { diff --git a/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java b/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java index a263c3a6903..7cb1d79080d 100644 --- a/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java +++ b/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java @@ -76,6 +76,24 @@ public static boolean waitWhileIsAlive(final long pid, Duration timeout, boolean return waitFor(() -> !isAlive(pid), timeout, printDots); } + /** + * Blocks until the pid file contains a new PID or timeout comes first. + * Doesn't check if the state of the process. + * + * @param oldPid process identifier + * @param pidFile file which will contain the new PID at some point + * @param timeout + * @param printDots true to print dots to STDOUT while waiting. One dot per second. + * @return true if the new PID was detected before timeout. False otherwise. + */ + public static boolean waitForNewPid(final long oldPid, final File pidFile, Duration timeout, boolean printDots) { + Supplier predicate = () -> { + final Long newPid = loadPid(pidFile); + return newPid != null && newPid.longValue() != oldPid; + }; + return waitFor(predicate, timeout, printDots); + } + /** * @param pidFile @@ -130,7 +148,7 @@ public static boolean isAlive(final ProcessHandle process) { * Blocks until the endpoint closes the connection or timeout comes first. * * @param endpoint endpoint host and port to use. - * @param timeout + * @param timeout must not be null * @param printDots true to print dots to STDOUT while waiting. One dot per second. * @return true if the connection was closed before timeout. False otherwise. */ diff --git a/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/xml/MiniXmlParser.java b/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/xml/MiniXmlParser.java index f0d62a62045..ac7485e7b64 100644 --- a/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/xml/MiniXmlParser.java +++ b/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/xml/MiniXmlParser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2024 Contributors to the Eclipse Foundation + * Copyright (c) 2022, 2025 Contributors to the Eclipse Foundation * Copyright (c) 2008, 2018 Oracle and/or its affiliates. All rights reserved. * * This program and the accompanying materials are made available under the @@ -173,8 +173,11 @@ public String getDomainName() { return domainName; } + /** + * @return never null + */ public List getAdminAddresses() { - if (adminAddresses == null || adminAddresses.isEmpty()) { + if (adminAddresses.isEmpty()) { String[] listenerNames = getListenerNamesForVS(DEFAULT_ADMIN_VS_ID, vsAttributes); if (listenerNames == null || listenerNames.length == 0) { listenerNames = getListenerNamesForVS(DEFAULT_VS_ID, vsAttributes); //plan B diff --git a/nucleus/common/common-util/src/main/java/com/sun/enterprise/util/io/ServerDirs.java b/nucleus/common/common-util/src/main/java/com/sun/enterprise/util/io/ServerDirs.java index 2698bc493dd..14a1e934d0f 100644 --- a/nucleus/common/common-util/src/main/java/com/sun/enterprise/util/io/ServerDirs.java +++ b/nucleus/common/common-util/src/main/java/com/sun/enterprise/util/io/ServerDirs.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2024 Contributors to the Eclipse Foundation + * Copyright (c) 2022, 2025 Contributors to the Eclipse Foundation * Copyright (c) 2010, 2020 Oracle and/or its affiliates. All rights reserved. * * This program and the accompanying materials are made available under the @@ -24,6 +24,7 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.nio.file.Path; import org.glassfish.main.jdke.i18n.LocalStringsImpl; @@ -213,6 +214,10 @@ public final boolean isValid() { return valid; } + public File getRestartLogFile() { + return getServerDir().toPath().resolve(Path.of("logs", "restart.log")).toFile(); + } + @Override public String toString() { return ObjectAnalyzer.toString(this); diff --git a/nucleus/common/common-util/src/main/java/com/sun/enterprise/util/net/NetUtils.java b/nucleus/common/common-util/src/main/java/com/sun/enterprise/util/net/NetUtils.java index dcd124cc0db..674bddb53e5 100644 --- a/nucleus/common/common-util/src/main/java/com/sun/enterprise/util/net/NetUtils.java +++ b/nucleus/common/common-util/src/main/java/com/sun/enterprise/util/net/NetUtils.java @@ -128,7 +128,6 @@ public static boolean isPortFree(String hostName, int portNumber) { if (portNumber <= 0 || portNumber > MAX_PORT) { return false; } - if (hostName == null || isThisHostLocal(hostName)) { return isPortFreeServer(portNumber); } @@ -136,26 +135,17 @@ public static boolean isPortFree(String hostName, int portNumber) { } private static boolean isPortFreeClient(String hostName, int portNumber) { - try { - // WBN - I have no idea why I'm messing with these streams! - // I lifted the code from installer. Apparently if you just - // open a socket on a free port and catch the exception something - // will go wrong in Windows. - // Feel free to change it if you know EXACTLY what you're doing - - //If the host name is null, assume localhost - if (hostName == null) { - hostName = getHostName(); - } - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(hostName, portNumber), IS_PORT_FREE_TIMEOUT); - } - } catch (Exception e) { + // If the host name is null, assume localhost + if (hostName == null) { + hostName = getHostName(); + } + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(hostName, portNumber), IS_PORT_FREE_TIMEOUT); + return false; + } catch (IOException e) { // Nobody is listening on this port - return true; } - - return false; + return true; } private static boolean isPortFreeServer(int port) { @@ -201,8 +191,8 @@ private static boolean isPortFreeServer(int port) { } } - private static boolean isPortFreeServer(int port, InetAddress add) { - try (ServerSocket ss = new ServerSocket(port, 10, add)) { + private static boolean isPortFreeServer(int port, InetAddress address) { + try (ServerSocket ss = new ServerSocket(port, 10, address)) { return true; } catch (Exception e) { return false; diff --git a/nucleus/core/bootstrap-osgi/src/main/java/org/glassfish/main/boot/embedded/AutoDisposableGlassFish.java b/nucleus/core/bootstrap-osgi/src/main/java/org/glassfish/main/boot/embedded/AutoDisposableGlassFish.java index c417b7b7c6a..c26ef2b65a8 100644 --- a/nucleus/core/bootstrap-osgi/src/main/java/org/glassfish/main/boot/embedded/AutoDisposableGlassFish.java +++ b/nucleus/core/bootstrap-osgi/src/main/java/org/glassfish/main/boot/embedded/AutoDisposableGlassFish.java @@ -72,6 +72,9 @@ class AutoDisposableGlassFish extends GlassFishImpl { if (commandRunner == null) { // only create the CommandRunner if needed commandRunner = serviceLocator.getService(CommandRunner.class); + if (commandRunner == null) { + throw new GlassFishException("Service locator failed to resolve the CommandRunner"); + } } String propertyPrefix = propertyName.split("\\.")[0]; if (!knownPropertyPrefixes.contains(propertyPrefix)) { diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/admin/StartServerHook.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/admin/StartServerHook.java index 90318f2d6a8..b713e64a247 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/admin/StartServerHook.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/admin/StartServerHook.java @@ -56,8 +56,6 @@ class StartServerShutdownHook extends Thread { private static final Logger LOG = System.getLogger(StartServerShutdownHook.class.getName()); private static final boolean LOG_RESTART = Boolean.parseBoolean(System.getenv("AS_RESTART_LOGFILES")); - private static final Path LOGDIR = new File(System.getProperty("com.sun.aas.instanceRoot"), "logs").toPath() - .toAbsolutePath(); private static final Predicate FILTER_OTHER_HOOKS = t -> t.getName().startsWith("GlassFish") && t.getName().endsWith("Shutdown Hook"); @@ -72,7 +70,7 @@ class StartServerShutdownHook extends Thread { throw new IllegalArgumentException("classname was null"); } this.startTime = Instant.now(); - this.logFile = getLogFileOld(startTime); + this.logFile = getLogFile(startTime); this.builder = prepareStartup(startTime, modulepath, classpath, sysProps, classname, args); } @@ -176,12 +174,13 @@ private void log(Exception e) { } - private static PrintStream getLogFileOld(Instant startTime) { + private static PrintStream getLogFile(Instant startTime) { if (!LOG_RESTART) { return null; } try { - return new PrintStream(LOGDIR.resolve("restart-" + startTime + "-old.log").toFile()); + return new PrintStream(new File(System.getProperty("com.sun.aas.instanceRoot"), "logs").toPath() + .resolve("start-server-shutdown-hook-" + startTime + ".log").toFile()); } catch (IOException e) { throw new IllegalStateException(e); } @@ -233,8 +232,7 @@ private static ProcessBuilder prepareStartup(Instant startTime, String modulepat outerCommand.add("-c"); // waitpid is not everywhere. outerCommand.add("(waitpid -e " + ProcessHandle.current().pid() + " || sleep 1) && '" - + cmdline.stream().collect(Collectors.joining("' '")) - + (LOG_RESTART ? "' > '" + LOGDIR.resolve("restart-" + startTime + "-new.log'") : "'")); + + cmdline.stream().collect(Collectors.joining("' '")) + "'"); } final ProcessBuilder builder = new ProcessBuilder(outerCommand); From 8e196fe42c19582333e138055d4aac19749d0f30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mat=C4=9Bj=C4=8Dek?= Date: Mon, 24 Nov 2025 12:07:27 +0100 Subject: [PATCH 02/10] The waitWhileListening did not work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - It declared the debug port free, but immediately 300 ms later it was blocked. Signed-off-by: David Matějček --- .../servermgmt/cli/LocalServerCommand.java | 8 ++-- .../servermgmt/cli/StartServerHelper.java | 7 ++- .../universal/process/ProcessUtils.java | 48 ------------------- 3 files changed, 10 insertions(+), 53 deletions(-) diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java index 6933f331f5a..68dff530bd9 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java @@ -49,7 +49,9 @@ import static com.sun.enterprise.admin.cli.CLIConstants.DEFAULT_HOSTNAME; import static com.sun.enterprise.admin.cli.ProgramOptions.PasswordLocation.LOCAL_PASSWORD; import static com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignChecker.step; +import static com.sun.enterprise.universal.process.ProcessUtils.isListening; import static com.sun.enterprise.universal.process.ProcessUtils.loadPid; +import static com.sun.enterprise.universal.process.ProcessUtils.waitFor; import static com.sun.enterprise.universal.process.ProcessUtils.waitForNewPid; import static com.sun.enterprise.universal.process.ProcessUtils.waitWhileIsAlive; import static com.sun.enterprise.util.SystemPropertyConstants.KEYSTORE_PASSWORD_DEFAULT; @@ -372,9 +374,9 @@ protected final void waitForStop(final Long pid, final HostAndPort adminAddress, return; } LOG.log(INFO, "Waiting until admin endpoint {0} is free.", adminAddress); - final boolean stopped = ProcessUtils.waitWhileListening(adminAddress, - portTimeout == null ? Duration.ofHours(1L) : portTimeout, printDots); - if (stopped) { + final boolean portIsFree = waitFor(() -> !isListening(adminAddress), portTimeout, printDots); + LOG.log(INFO, "Admin port is {0}.", portIsFree ? "free" : "blocked"); + if (portIsFree) { return; } throw new CommandException("Timed out waiting for the server to stop."); diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java index 9817e63d1c6..f68b4846f6c 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java @@ -51,7 +51,8 @@ import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_DEBUG_ON; import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_NORMAL; import static com.sun.enterprise.admin.cli.CLIConstants.WALL_CLOCK_START_PROP; -import static com.sun.enterprise.universal.process.ProcessUtils.waitWhileListening; +import static com.sun.enterprise.universal.process.ProcessUtils.isListening; +import static com.sun.enterprise.universal.process.ProcessUtils.waitFor; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.INFO; import static org.glassfish.main.jdke.props.SystemProperties.setProperty; @@ -96,7 +97,9 @@ public StartServerHelper(boolean terse, Duration timeout, ServerDirs serverDirs, final Integer debugPort = launcher.getDebugPort(); if (debugPort != null) { LOG.log(INFO, "Waiting few seconds until debug port {0} is free.", debugPort); - waitWhileListening(new HostAndPort("localhost", debugPort, true), Duration.ofSeconds(10), terse); + final HostAndPort debugEndpoint = new HostAndPort("localhost", debugPort, false); + final boolean portIsFree = waitFor(() -> !isListening(debugEndpoint), timeout, terse); + LOG.log(INFO, "Debug port is {0}.", portIsFree ? "free" : "blocked"); } } checkFreeAdminPorts(info.getAdminAddresses()); diff --git a/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java b/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java index 7cb1d79080d..071a24111ce 100644 --- a/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java +++ b/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java @@ -28,8 +28,6 @@ import java.lang.System.Logger; import java.net.InetSocketAddress; import java.net.Socket; -import java.net.SocketException; -import java.net.SocketTimeoutException; import java.text.MessageFormat; import java.time.Duration; import java.time.Instant; @@ -38,10 +36,8 @@ import static com.sun.enterprise.util.StringUtils.ok; import static java.lang.System.Logger.Level.DEBUG; -import static java.lang.System.Logger.Level.ERROR; import static java.lang.System.Logger.Level.INFO; import static java.lang.System.Logger.Level.TRACE; -import static java.lang.System.Logger.Level.WARNING; import static java.nio.charset.StandardCharsets.ISO_8859_1; /** @@ -144,50 +140,6 @@ public static boolean isAlive(final ProcessHandle process) { return true; } - /** - * Blocks until the endpoint closes the connection or timeout comes first. - * - * @param endpoint endpoint host and port to use. - * @param timeout must not be null - * @param printDots true to print dots to STDOUT while waiting. One dot per second. - * @return true if the connection was closed before timeout. False otherwise. - */ - public static boolean waitWhileListening(HostAndPort endpoint, Duration timeout, boolean printDots) { - final DotPrinter dotPrinter = DotPrinter.startWaiting(printDots); - try (Socket server = new Socket()) { - server.setSoTimeout((int) timeout.toMillis()); - // Max 5 seconds to connect. It is an extreme value for local endpoint. - try { - server.connect(new InetSocketAddress(endpoint.getHost(), endpoint.getPort()), SOCKET_TIMEOUT); - } catch (IOException e) { - LOG.log(TRACE, "Unable to connect - server is probably down.!", e); - return true; - } - try { - int result = server.getInputStream().read(); - if (result == -1) { - LOG.log(TRACE, "Input stream closed - server probably stopped!"); - return true; - } - LOG.log(ERROR, "We were able to read something: {0}. Returning false.", result); - return false; - } catch (SocketTimeoutException e) { - LOG.log(TRACE, "Timeout while reading. Returning false.", e); - return false; - } catch (SocketException e) { - LOG.log(TRACE, "Socket read failed. Returning true.", e); - return true; - } - } catch (Exception ex) { - LOG.log(WARNING, "An attempt to open a socket to " + endpoint - + " resulted in exception. Therefore we assume the server has stopped.", ex); - return true; - } finally { - DotPrinter.stopWaiting(dotPrinter); - } - } - - /** * @param endpoint endpoint host and port to use. * @return true if the endpoint is listening on socket From bc8798d454eaa3aefbe5c713fe01b969f08eb52d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mat=C4=9Bj=C4=8Dek?= Date: Tue, 25 Nov 2025 05:30:25 +0100 Subject: [PATCH 03/10] Fixed ORB cleanup and using ports after fast restarts (reuse) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Matějček --- .../iiop/api/GlassFishORBHelper.java | 36 ++++++++++++------- .../iiop/impl/IIOPSSLSocketFactory.java | 16 ++++----- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/appserver/orb/orb-connector/src/main/java/org/glassfish/enterprise/iiop/api/GlassFishORBHelper.java b/appserver/orb/orb-connector/src/main/java/org/glassfish/enterprise/iiop/api/GlassFishORBHelper.java index 5b2d16b4d62..70f7a72139e 100644 --- a/appserver/orb/orb-connector/src/main/java/org/glassfish/enterprise/iiop/api/GlassFishORBHelper.java +++ b/appserver/orb/orb-connector/src/main/java/org/glassfish/enterprise/iiop/api/GlassFishORBHelper.java @@ -17,21 +17,19 @@ package org.glassfish.enterprise.iiop.api; -import com.sun.logging.LogDomains; - import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; import jakarta.ejb.Singleton; import jakarta.inject.Inject; import jakarta.inject.Provider; +import java.lang.System.Logger; import java.nio.channels.SelectableChannel; import java.rmi.Remote; import java.util.Properties; -import java.util.logging.Level; -import java.util.logging.Logger; import org.glassfish.api.admin.ProcessEnvironment; +import org.glassfish.api.event.EventListener; +import org.glassfish.api.event.Events; import org.glassfish.api.naming.GlassfishNamingManager; import org.glassfish.hk2.api.ServiceLocator; import org.glassfish.internal.api.ORBLocator; @@ -39,7 +37,8 @@ import org.omg.CORBA.ORB; import org.omg.PortableInterceptor.ServerRequestInfo; -import static com.sun.logging.LogDomains.CORBA_LOGGER; +import static java.lang.System.Logger.Level.INFO; +import static org.glassfish.api.event.EventTypes.SERVER_SHUTDOWN; /** * This class exposes any orb/iiop functionality needed by modules in the app server. @@ -51,7 +50,10 @@ @Singleton public class GlassFishORBHelper implements ORBLocator { - private static final Logger LOG = LogDomains.getLogger(GlassFishORBHelper.class, CORBA_LOGGER, false); + private static final Logger LOG = System.getLogger(GlassFishORBHelper.class.getName()); + + @Inject + private Provider eventsProvider; @Inject private ServiceLocator services; @@ -75,17 +77,27 @@ public class GlassFishORBHelper implements ORBLocator { @PostConstruct public void postConstruct() { + // WARN: Neither PreDestroy annotation nor interface worked! + EventListener glassfishEventListener = event -> { + if (event.is(SERVER_SHUTDOWN)) { + onShutdown(); + } + }; + eventsProvider.get().register(glassfishEventListener); orbFactory = services.getService(GlassFishORBFactory.class); + LOG.log(INFO, "GlassFishORBLocator created."); } - @PreDestroy - public void onShutdown() { + private void onShutdown() { // FIXME: getORB is able to create another, it should be refactored and simplified. destroyed = true; - LOG.log(Level.CONFIG, "ORB Shutdown started"); - if (orb != null) { - orb.destroy(); + LOG.log(INFO, "ORB shutdown started"); + if (this.orb != null) { + // First remove, then destroy. + // Still, threads already working with the instance will have it unstable. + final ORB destroyedOrb = orb; orb = null; + destroyedOrb.destroy(); } } diff --git a/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java b/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java index 2674c4de8e5..18d07ef68f3 100644 --- a/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java +++ b/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java @@ -251,13 +251,9 @@ public void setORB(ORB orb) { */ @Override public ServerSocket createServerSocket(String type, InetSocketAddress inetSocketAddress) throws IOException { - if (LOG.isLoggable(Level.FINE)) { - LOG.log(Level.FINE, "Creating server socket for type =" + type - + " inetSocketAddress =" + inetSocketAddress); - } + LOG.log(Level.INFO, "Creating server socket for type =" + type + " inetSocketAddress =" + inetSocketAddress); - if(type.equals(SSL_MUTUALAUTH) || type.equals(SSL) || - type.equals(PERSISTENT_SSL)) { + if (type.equals(SSL_MUTUALAUTH) || type.equals(SSL) || type.equals(PERSISTENT_SSL)) { return createSSLServerSocket(type, inetSocketAddress); } ServerSocket serverSocket = null; @@ -267,9 +263,9 @@ public ServerSocket createServerSocket(String type, InetSocketAddress inetSocket } else { serverSocket = new ServerSocket(); } - - serverSocket.bind(inetSocketAddress); - return serverSocket; + serverSocket.setReuseAddress(true); + serverSocket.bind(inetSocketAddress); + return serverSocket; } /** @@ -360,12 +356,14 @@ private ServerSocket createSSLServerSocket(String type, InetSocketAddress inetSo LOG.log(Level.FINE, "Cipher Suite: " + element); } } + ServerSocket ss = null; try { // bugfix for 6349541 // specify the ip address to bind to, 50 is the default used // by the ssf implementation when only the port is specified ss = ssf.createServerSocket(port, BACKLOG, inetSocketAddress.getAddress()); + ss.setReuseAddress(true); if (ciphers != null) { ((SSLServerSocket) ss).setEnabledCipherSuites(ciphers); } From 2f2ceacff3cc5a8c753cda0d95522fc9ba485c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mat=C4=9Bj=C4=8Dek?= Date: Tue, 25 Nov 2025 06:29:25 +0100 Subject: [PATCH 04/10] Added port checking to ORB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Matějček --- .../iiop/impl/IIOPSSLSocketFactory.java | 24 +++++++++++++++---- .../universal/process/ProcessUtils.java | 1 + 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java b/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java index 18d07ef68f3..edc863fbe25 100644 --- a/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java +++ b/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java @@ -23,6 +23,8 @@ import com.sun.corba.ee.spi.transport.ORBSocketFactory; import com.sun.enterprise.config.serverbeans.Config; import com.sun.enterprise.security.integration.AppClientSSL; +import com.sun.enterprise.universal.process.ProcessUtils; +import com.sun.enterprise.util.HostAndPort; import com.sun.logging.LogDomains; import java.io.IOException; @@ -32,6 +34,7 @@ import java.net.SocketException; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; +import java.time.Duration; import java.util.ArrayList; import java.util.Hashtable; import java.util.List; @@ -89,7 +92,7 @@ public class IIOPSSLSocketFactory implements ORBSocketFactory { * @todo provide an interface to the admin, so that whenever a iiop-listener * is added / removed, we modify the hashtable, */ - private final Map portToSSLInfo = new Hashtable(); + private final Map portToSSLInfo = new Hashtable<>(); /* this is stored for the client side of SSL Connections. * Note: There will be only 1 ctx for the client side, as we will reuse the * ctx for all SSL connections @@ -175,8 +178,7 @@ public IIOPSSLSocketFactory() { } } } catch (Exception e) { - LOG.log(Level.SEVERE,"IIOPSSLSocketFactory initialization failed.", e); - throw new IllegalStateException(e); + throw new IllegalStateException("IIOPSSLSocketFactory initialization failed.", e); } } @@ -256,7 +258,7 @@ public ServerSocket createServerSocket(String type, InetSocketAddress inetSocket if (type.equals(SSL_MUTUALAUTH) || type.equals(SSL) || type.equals(PERSISTENT_SSL)) { return createSSLServerSocket(type, inetSocketAddress); } - ServerSocket serverSocket = null; + final ServerSocket serverSocket; if (orb.getORBData().acceptorSocketType().equals(SOCKETCHANNEL)) { ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); serverSocket = serverSocketChannel.socket(); @@ -264,6 +266,7 @@ public ServerSocket createServerSocket(String type, InetSocketAddress inetSocket serverSocket = new ServerSocket(); } serverSocket.setReuseAddress(true); + checkPort(inetSocketAddress); serverSocket.bind(inetSocketAddress); return serverSocket; } @@ -335,7 +338,7 @@ private ServerSocket createSSLServerSocket(String type, InetSocketAddress inetSo } int port = inetSocketAddress.getPort(); Integer iport = Integer.valueOf(port); - SSLInfo sslInfo = (SSLInfo)portToSSLInfo.get(iport); + SSLInfo sslInfo = portToSSLInfo.get(iport); if (sslInfo == null) { throw new IOException("No SSL info found for port " + iport); } @@ -362,6 +365,7 @@ private ServerSocket createSSLServerSocket(String type, InetSocketAddress inetSo // bugfix for 6349541 // specify the ip address to bind to, 50 is the default used // by the ssf implementation when only the port is specified + checkPort(inetSocketAddress); ss = ssf.createServerSocket(port, BACKLOG, inetSocketAddress.getAddress()); ss.setReuseAddress(true); if (ciphers != null) { @@ -388,6 +392,16 @@ private ServerSocket createSSLServerSocket(String type, InetSocketAddress inetSo return ss; } + /** FIXME Temporary hack until we find out which part is leaking. */ + private static void checkPort(InetSocketAddress address) { + int port = address.getPort(); + if (port < 1) { + return; + } + HostAndPort endpoint = new HostAndPort(address.getHostString(), port, false); + ProcessUtils.waitFor(() -> !ProcessUtils.isListening(endpoint), Duration.ofSeconds(10L), true); + } + /** * Create an SSL socket at the specified host and port. * @param host diff --git a/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java b/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java index 071a24111ce..f586d244472 100644 --- a/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java +++ b/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java @@ -146,6 +146,7 @@ public static boolean isAlive(final ProcessHandle process) { */ public static boolean isListening(HostAndPort endpoint) { try (Socket server = new Socket()) { + server.setReuseAddress(false); // Max 5 seconds to connect. It is an extreme value for local endpoint. server.connect(new InetSocketAddress(endpoint.getHost(), endpoint.getPort()), SOCKET_TIMEOUT); return true; From d62136f748d9fea246ac04c5b824bee761b1692c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mat=C4=9Bj=C4=8Dek?= Date: Tue, 25 Nov 2025 21:47:08 +0100 Subject: [PATCH 05/10] Improved port checking, reduced logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Matějček --- .../iiop/impl/IIOPSSLSocketFactory.java | 9 +- .../enterprise/admin/launcher/GFLauncher.java | 8 -- .../servermgmt/cli/LocalServerCommand.java | 7 +- .../admin/servermgmt/cli/PortWatcher.java | 88 ++++++++++++++ .../servermgmt/cli/RestartDomainCommand.java | 18 ++- .../servermgmt/cli/ServerLifeSignChecker.java | 4 +- .../servermgmt/cli/StartServerHelper.java | 29 +++-- .../ChangeNodeMasterPasswordCommand.java | 7 +- .../cluster/RestartLocalInstanceCommand.java | 19 ++- .../universal/process/ProcessUtils.java | 115 +++++++++++++++--- .../com/sun/enterprise/util/HostAndPort.java | 10 +- .../glassfish/tests/utils/ServerUtils.java | 1 - 12 files changed, 252 insertions(+), 63 deletions(-) create mode 100644 nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/PortWatcher.java diff --git a/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java b/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java index edc863fbe25..7c326aa2612 100644 --- a/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java +++ b/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java @@ -265,7 +265,6 @@ public ServerSocket createServerSocket(String type, InetSocketAddress inetSocket } else { serverSocket = new ServerSocket(); } - serverSocket.setReuseAddress(true); checkPort(inetSocketAddress); serverSocket.bind(inetSocketAddress); return serverSocket; @@ -367,7 +366,6 @@ private ServerSocket createSSLServerSocket(String type, InetSocketAddress inetSo // by the ssf implementation when only the port is specified checkPort(inetSocketAddress); ss = ssf.createServerSocket(port, BACKLOG, inetSocketAddress.getAddress()); - ss.setReuseAddress(true); if (ciphers != null) { ((SSLServerSocket) ss).setEnabledCipherSuites(ciphers); } @@ -398,8 +396,11 @@ private static void checkPort(InetSocketAddress address) { if (port < 1) { return; } - HostAndPort endpoint = new HostAndPort(address.getHostString(), port, false); - ProcessUtils.waitFor(() -> !ProcessUtils.isListening(endpoint), Duration.ofSeconds(10L), true); + final HostAndPort endpoint = new HostAndPort(address.getHostString(), port, false); + if (!ProcessUtils.isListening(endpoint)) { + return; + } + ProcessUtils.waitWhileListening(endpoint, Duration.ofSeconds(10L), false); } /** diff --git a/nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java b/nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java index d55e20883c0..34d271d4f4f 100644 --- a/nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java +++ b/nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java @@ -21,10 +21,8 @@ import com.sun.enterprise.universal.glassfish.GFLauncherUtils; import com.sun.enterprise.universal.glassfish.TokenResolver; import com.sun.enterprise.universal.process.ProcessStreamDrainer; -import com.sun.enterprise.universal.process.ProcessUtils; import com.sun.enterprise.universal.xml.MiniXmlParser; import com.sun.enterprise.universal.xml.MiniXmlParserException; -import com.sun.enterprise.util.HostAndPort; import com.sun.enterprise.util.io.FileUtils; import java.io.BufferedWriter; @@ -206,12 +204,6 @@ public abstract class GFLauncher { */ public final void launch() throws GFLauncherException { if (debugPort != null) { - // As the debugger starts immediately with the new process, - // and especially if there is some problem with shutdown, we can escalate it. - if (ProcessUtils.isListening(new HostAndPort("localhost", debugPort, false))) { - throw new GFLauncherException( - "The debug port " + debugPort + " is already occupied by another process."); - } if (debugSuspend) { LOG.log(INFO, () -> "Debugging will be available and the server will start suspended on port " + debugPort + "."); diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java index 68dff530bd9..527502b6a7e 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java @@ -49,11 +49,10 @@ import static com.sun.enterprise.admin.cli.CLIConstants.DEFAULT_HOSTNAME; import static com.sun.enterprise.admin.cli.ProgramOptions.PasswordLocation.LOCAL_PASSWORD; import static com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignChecker.step; -import static com.sun.enterprise.universal.process.ProcessUtils.isListening; import static com.sun.enterprise.universal.process.ProcessUtils.loadPid; -import static com.sun.enterprise.universal.process.ProcessUtils.waitFor; import static com.sun.enterprise.universal.process.ProcessUtils.waitForNewPid; import static com.sun.enterprise.universal.process.ProcessUtils.waitWhileIsAlive; +import static com.sun.enterprise.universal.process.ProcessUtils.waitWhileListening; import static com.sun.enterprise.util.SystemPropertyConstants.KEYSTORE_PASSWORD_DEFAULT; import static com.sun.enterprise.util.SystemPropertyConstants.MASTER_PASSWORD_ALIAS; import static com.sun.enterprise.util.SystemPropertyConstants.MASTER_PASSWORD_FILENAME; @@ -373,9 +372,7 @@ protected final void waitForStop(final Long pid, final HostAndPort adminAddress, if (adminAddress == null) { return; } - LOG.log(INFO, "Waiting until admin endpoint {0} is free.", adminAddress); - final boolean portIsFree = waitFor(() -> !isListening(adminAddress), portTimeout, printDots); - LOG.log(INFO, "Admin port is {0}.", portIsFree ? "free" : "blocked"); + final boolean portIsFree = waitWhileListening(adminAddress, portTimeout, printDots); if (portIsFree) { return; } diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/PortWatcher.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/PortWatcher.java new file mode 100644 index 00000000000..1dd7829dcb6 --- /dev/null +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/PortWatcher.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2025 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0, which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the + * Eclipse Public License v. 2.0 are satisfied: GNU General Public License, + * version 2 with the GNU Classpath Exception, which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + */ + +package com.sun.enterprise.admin.servermgmt.cli; + +import com.sun.enterprise.universal.process.ProcessUtils; +import com.sun.enterprise.util.HostAndPort; + +import java.lang.System.Logger; +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; + +import static java.lang.System.Logger.Level.TRACE; + + +/** + * Use case: + *
    + *
  1. We have a running server listening on some endpoint + *
  2. We will watch it until it closes the endpoint. + *
  3. In parallel to the watching we ask the server to stop. + *
+ * Why - if we would start watching after the command, endpoint could be already in some state + * when it would refuse new connections, but it would also refuse to rebind on the server side + * of the new process. So we need to be sure the endpoint stopped listening before we start + * binding again. + */ +public class PortWatcher { + private static final Logger LOG = System.getLogger(PortWatcher.class.getName()); + + private final CompletableFuture job; + + private PortWatcher(Supplier supplier) { + this.job = CompletableFuture.supplyAsync(supplier); + // We should be always sure we do listen to the old process and not to the new one. + // If it starts to be flaky, replace with sleep. + Thread.onSpinWait(); + } + + /** + * Blocks until we have the positive answer or endpoint is still listening after timeout. + * + * @param timeout can be null, then we may wait forever. + * @return true if the endpoint disconnected before timeout, false if it timed out. + */ + public boolean get(Duration timeout) { + LOG.log(TRACE, "get(timeout={0})", timeout); + try { + if (timeout == null) { + return job.get(); + } + return job.get(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + job.cancel(true); + return false; + } + } + + /** + * Watch the endpoint in a separate thread. + * + * @param endpoint + * @param printDots + * @return the {@link PortWatcher} + */ + public static PortWatcher watch(HostAndPort endpoint, boolean printDots) { + Objects.requireNonNull(endpoint, "endpoint"); + return new PortWatcher(() -> ProcessUtils.waitWhileListening(endpoint, null, printDots)); + } +} diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/RestartDomainCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/RestartDomainCommand.java index 10167b13f30..4893dc60662 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/RestartDomainCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/RestartDomainCommand.java @@ -32,7 +32,6 @@ import org.glassfish.api.admin.CommandException; import org.glassfish.hk2.api.PerLookup; import org.glassfish.hk2.api.ServiceLocator; -import org.glassfish.main.jdke.i18n.LocalStringsImpl; import org.jvnet.hk2.annotations.Service; import static com.sun.enterprise.admin.cli.CLIConstants.DEATH_TIMEOUT_MS; @@ -58,7 +57,6 @@ @Service(name = "restart-domain") @PerLookup public class RestartDomainCommand extends StopDomainCommand { - private static final LocalStringsImpl I18N = new LocalStringsImpl(RestartDomainCommand.class); @Param(name = "debug", optional = true) private Boolean debug; @@ -97,7 +95,8 @@ protected void doCommand() throws CommandException { // oldPid is received from the running server. final Long oldPid = getServerPid(); final HostAndPort oldAdminAddress = getReachableAdminAddress(); - final HostAndPort newAdminEndpoint = getAdminAddress("server"); + final boolean printDots = !programOpts.isTerse(); + final PortWatcher portWatcher = oldAdminAddress == null ? null : PortWatcher.watch(oldAdminAddress, printDots); final RemoteCLICommand cmd = new RemoteCLICommand("restart-domain", programOpts, env); if (debug == null) { cmd.executeAndReturnOutput("restart-domain"); @@ -106,8 +105,16 @@ protected void doCommand() throws CommandException { } final Duration timeout = getRestartTimeout(); - final Duration startTimeout = step(null, timeout, - () -> waitForStop(isLocal() ? oldPid : null, oldAdminAddress, timeout)); + final Duration startTimeout; + if (isLocal()) { + startTimeout = step(null, timeout, () -> waitForStop(oldPid, null, timeout)); + } else { + startTimeout = timeout; + } + + if (portWatcher != null && !portWatcher.get(startTimeout)) { + logger.warning("The endpoint is still listening after timeout: " + oldAdminAddress); + } final List userEndpoints = parseCustomEndpoints(customEndpoints); final ServerLifeSignCheck lifeSignCheck = new ServerLifeSignCheck("domain " + getDomainName(), @@ -117,6 +124,7 @@ protected void doCommand() throws CommandException { logger.info(report); } + /** * If the server isn't running, try to start it. */ diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignChecker.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignChecker.java index 1149e78ea88..b17f3afe359 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignChecker.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignChecker.java @@ -29,7 +29,7 @@ import org.glassfish.api.admin.CommandException; -import static java.lang.System.Logger.Level.INFO; +import static java.lang.System.Logger.Level.DEBUG; public class ServerLifeSignChecker { private static final Logger LOG = System.getLogger(ServerLifeSignChecker.class.getName()); @@ -223,7 +223,7 @@ public static Duration step(String message, Duration timeout, Action action) thr return timeout; } if (message != null) { - LOG.log(INFO, message); + LOG.log(DEBUG, message); } Instant start = Instant.now(); action.action(); diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java index f68b4846f6c..ea6aac70ffa 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java @@ -51,8 +51,6 @@ import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_DEBUG_ON; import static com.sun.enterprise.admin.cli.CLIConstants.RESTART_NORMAL; import static com.sun.enterprise.admin.cli.CLIConstants.WALL_CLOCK_START_PROP; -import static com.sun.enterprise.universal.process.ProcessUtils.isListening; -import static com.sun.enterprise.universal.process.ProcessUtils.waitFor; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.INFO; import static org.glassfish.main.jdke.props.SystemProperties.setProperty; @@ -94,14 +92,8 @@ public StartServerHelper(boolean terse, Duration timeout, ServerDirs serverDirs, if (launcher.getPidBeforeRestart() != null) { waitForParentToDie(launcher.getPidBeforeRestart(), timeout); configureLoggingOfRestart(serverDirs.getRestartLogFile()); - final Integer debugPort = launcher.getDebugPort(); - if (debugPort != null) { - LOG.log(INFO, "Waiting few seconds until debug port {0} is free.", debugPort); - final HostAndPort debugEndpoint = new HostAndPort("localhost", debugPort, false); - final boolean portIsFree = waitFor(() -> !isListening(debugEndpoint), timeout, terse); - LOG.log(INFO, "Debug port is {0}.", portIsFree ? "free" : "blocked"); - } } + checkFreeDebugPort(launcher.getDebugPort(), Duration.ofSeconds(10L), terse); checkFreeAdminPorts(info.getAdminAddresses()); deletePidFile(); } @@ -205,7 +197,7 @@ private void configureLoggingOfRestart(File logFile) { * * @throws CommandException if we timeout waiting for the parent to die or if the admin ports never free up */ - private void waitForParentToDie(long pid, Duration timeout) throws GFLauncherException { + private void waitForParentToDie(Long pid, Duration timeout) throws GFLauncherException { LOG.log(INFO, () -> "Waiting for death of the parent process with the pid " + pid); if (!ProcessUtils.waitWhileIsAlive(pid, timeout, false)) { throw new GFLauncherException("Waited " + timeout.toSeconds() @@ -280,8 +272,23 @@ public static List parseCustomEndpoints(String customEndpoints) thr return endpoints; } + /** + * Fast respawn can meet with previous JVM on ports, despite the JVM is already dead. + * So we have to wait a bit. + */ + private static void checkFreeDebugPort(Integer debugPort, Duration timeout, boolean terse) { + if (debugPort == null) { + return; + } + final HostAndPort debugEndpoint = new HostAndPort("localhost", debugPort, false); + if (!ProcessUtils.isListening(debugEndpoint)) { + return; + } + ProcessUtils.waitWhileListening(debugEndpoint, timeout, !terse); + } + private static void checkFreeAdminPorts(List endpoints) throws GFLauncherException { - LOG.log(INFO, "Checking if all admin ports are free."); + LOG.log(DEBUG, "Checking if all admin ports are free."); for (HostAndPort endpoint : endpoints) { if (!NetUtils.isPortFree(endpoint.getHost(), endpoint.getPort())) { throw new GFLauncherException("There is a process already using the admin port " + endpoint.getPort() diff --git a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/ChangeNodeMasterPasswordCommand.java b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/ChangeNodeMasterPasswordCommand.java index 45d45356484..d7478a1bad2 100644 --- a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/ChangeNodeMasterPasswordCommand.java +++ b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/ChangeNodeMasterPasswordCommand.java @@ -83,9 +83,8 @@ protected int executeCommand() throws CommandException { ArrayList serverNames = getInstanceDirs(serverDir); for (String serverName: serverNames) { - if (isRunning(serverDir, serverName)) { - throw new CommandException(strings.get("instance.is.running", - serverName)); + if (isRunning(serverName)) { + throw new CommandException(strings.get("instance.is.running", serverName)); } } @@ -213,7 +212,7 @@ private ArrayList getInstanceDirs(File parent) throws CommandException { } - private boolean isRunning(File nodeDirChild, String serverName) throws CommandException { + private boolean isRunning(String serverName) throws CommandException { File serverDir = new File(nodeDirChild, serverName); File configDir = new File(serverDir, "config"); File domainXml = new File(configDir, "domain.xml"); diff --git a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/RestartLocalInstanceCommand.java b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/RestartLocalInstanceCommand.java index c62444e457b..341f6233e49 100644 --- a/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/RestartLocalInstanceCommand.java +++ b/nucleus/cluster/cli/src/main/java/com/sun/enterprise/admin/cli/cluster/RestartLocalInstanceCommand.java @@ -19,6 +19,7 @@ import com.sun.enterprise.admin.cli.CLICommand; import com.sun.enterprise.admin.cli.remote.RemoteCLICommand; +import com.sun.enterprise.admin.servermgmt.cli.PortWatcher; import com.sun.enterprise.admin.servermgmt.cli.ServerLifeSignCheck; import com.sun.enterprise.util.HostAndPort; @@ -63,9 +64,9 @@ protected final void doCommand() throws CommandException { // Save old values before executing restart final Long oldPid = getServerPid(); final HostAndPort oldAdminAddress = getReachableAdminAddress(); - - // run the remote restart-instance command and throw away the output - RemoteCLICommand cmd = new RemoteCLICommand("_restart-instance", programOpts, env); + final boolean printDots = !programOpts.isTerse(); + final PortWatcher portWatcher = oldAdminAddress == null ? null : PortWatcher.watch(oldAdminAddress, printDots); + final RemoteCLICommand cmd = new RemoteCLICommand("_restart-instance", programOpts, env); if (debug == null) { cmd.executeAndReturnOutput("_restart-instance"); } else { @@ -73,8 +74,16 @@ protected final void doCommand() throws CommandException { } final Duration timeout = getRestartTimeout(); - final Duration startTimeout = step("Waiting until instance stops.", timeout, - () -> waitForStop(oldPid, oldAdminAddress, timeout)); + final Duration startTimeout; + if (isLocal()) { + startTimeout = step(null, timeout, () -> waitForStop(oldPid, null, timeout)); + } else { + startTimeout = timeout; + } + + if (portWatcher != null && !portWatcher.get(startTimeout)) { + logger.warning("The endpoint is still listening after timeout: " + oldAdminAddress); + } final ServerLifeSignCheck lifeSignCheck = new ServerLifeSignCheck("instance " + getInstanceName(), true, true, true, true, List.of()); final String report = waitForStart(oldPid, lifeSignCheck, () -> List.of(getReachableAdminAddress()), startTimeout); diff --git a/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java b/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java index f586d244472..18209db42c7 100644 --- a/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java +++ b/nucleus/common/common-util/src/main/java/com/sun/enterprise/universal/process/ProcessUtils.java @@ -28,6 +28,12 @@ import java.lang.System.Logger; import java.net.InetSocketAddress; import java.net.Socket; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.nio.channels.ClosedByInterruptException; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; import java.text.MessageFormat; import java.time.Duration; import java.time.Instant; @@ -38,6 +44,7 @@ import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.INFO; import static java.lang.System.Logger.Level.TRACE; +import static java.lang.System.Logger.Level.WARNING; import static java.nio.charset.StandardCharsets.ISO_8859_1; /** @@ -52,7 +59,8 @@ public final class ProcessUtils { private static final Logger LOG = System.getLogger(ProcessUtils.class.getName()); - private static final int SOCKET_TIMEOUT = 5000; + /** 1 second is long enough for local connection */ + private static final int SOCKET_CONNECT_TIMEOUT = 1000; private static final String[] PATH = getSystemPath(); private ProcessUtils() { @@ -140,24 +148,94 @@ public static boolean isAlive(final ProcessHandle process) { return true; } + /** + * Blocks until the endpoint closes the connection or timeout comes first. + *

+ * The important difference between this and {@link #isListening(HostAndPort)} is that + * this method connects to an existing endpoint and stays connected until the endpoint + * really disconnects.
+ * If you would combine {@link #waitFor(Supplier, Duration, boolean)} and + * negated {@link #isListening(HostAndPort)}, your connection could be refused, but + * that doesn't mean that another process is free to bind with the port again. + *

+ * This method is not suitable for endpoints in an unknown state, because it would + * then use whole connect timeout if the endpoint is not listening. + * + * @param endpoint endpoint host and port to use. + * @param timeout must not be null + * @param printDots true to print dots to STDOUT while waiting. One dot per second. + * @return true if the connection was closed before timeout. False otherwise. + */ + public static boolean waitWhileListening(HostAndPort endpoint, Duration timeout, boolean printDots) { + final Supplier action = () -> { + try (Socket server = new Socket()) { + server.setSoTimeout(timeout == null ? 0 : (int) timeout.toMillis()); + try { + server.connect(endpoint.toInetSocketAddress(), SOCKET_CONNECT_TIMEOUT); + } catch (IOException e) { + LOG.log(TRACE, "Unable to connect - server is probably down.!", e); + return true; + } + while (true) { + try { + int result = server.getInputStream().read(); + if (result == -1) { + LOG.log(TRACE, "Input stream closed - server probably stopped!"); + return true; + } + LOG.log(TRACE, "We were able to read something: {0}. Continuing.", result); + } catch (SocketTimeoutException | ClosedByInterruptException e) { + LOG.log(TRACE, "Socket read waiting timed out; returning false.", e); + return false; + } catch (SocketException e) { + LOG.log(TRACE, "Socket read waiting finished; returning true.", e); + return true; + } + } + } catch (IOException ex) { + LOG.log(WARNING, "An attempt to open a socket to " + endpoint + + " resulted in exception. Therefore we assume the server has stopped.", ex); + return true; + } + }; + LOG.log(DEBUG, () -> "Waiting until endpoint " + endpoint + " stops listening."); + final DotPrinter dotPrinter = DotPrinter.startWaiting(printDots); + final Instant start = Instant.now(); + final boolean result = action.get(); + DotPrinter.stopWaiting(dotPrinter); + final long millis = Duration.between(start, Instant.now()).toMillis(); + LOG.log(result ? DEBUG : WARNING, () -> "Waiting finished after " + millis + " ms. Endpoint " + endpoint + + (result ? " stopped" : " did not stop") + " listening."); + return result; + } + /** * @param endpoint endpoint host and port to use. * @return true if the endpoint is listening on socket */ public static boolean isListening(HostAndPort endpoint) { - try (Socket server = new Socket()) { - server.setReuseAddress(false); - // Max 5 seconds to connect. It is an extreme value for local endpoint. - server.connect(new InetSocketAddress(endpoint.getHost(), endpoint.getPort()), SOCKET_TIMEOUT); - return true; - } catch (Exception ex) { + try (SocketChannel channel = SocketChannel.open()) { + channel.configureBlocking(false); + channel.connect(new InetSocketAddress(endpoint.getHost(), endpoint.getPort())); + + try (Selector selector = Selector.open()) { + channel.register(selector, SelectionKey.OP_CONNECT); + if (selector.select(SOCKET_CONNECT_TIMEOUT) == 0) { + // Timeout + return false; + } + channel.finishConnect(); + // Successfully connected = port is listening + return true; + } + } catch (IOException e) { LOG.log(TRACE, "An attempt to open a socket to " + endpoint - + " resulted in exception. Therefore we assume the server has stopped.", ex); + + " resulted in exception. Therefore we assume the server has stopped.", e); + // Connection failed return false; } } - /** * Kill the process with the given Process ID and wait until it's gone - that means * that the watchedPidFile is deleted OR the process is not resolved as alive by @@ -209,10 +287,9 @@ public static void kill(File pidFile, Duration timeout, boolean printDots) */ public static boolean waitFor(Supplier sign, Duration timeout, boolean printDots) { LOG.log(DEBUG, "waitFor(sign={0}, timeout={1}, printDots={2})", sign, timeout, printDots); - final DotPrinter dotPrinter = DotPrinter.startWaiting(printDots); final Instant start = Instant.now(); - try { - final Instant deadline = timeout == null ? null : start.plus(timeout); + final Instant deadline = timeout == null ? null : start.plus(timeout); + final Supplier action = () -> { while (deadline == null || Instant.now().isBefore(deadline)) { if (sign.get()) { return true; @@ -220,10 +297,14 @@ public static boolean waitFor(Supplier sign, Duration timeout, boolean Thread.onSpinWait(); } return false; - } finally { - DotPrinter.stopWaiting(dotPrinter); - LOG.log(INFO, "Waiting finished after {0} ms.", Duration.between(start, Instant.now()).toMillis()); - } + }; + final DotPrinter dotPrinter = DotPrinter.startWaiting(printDots); + final Boolean result = action.get(); + DotPrinter.stopWaiting(dotPrinter); + final long millis = Duration.between(start, Instant.now()).toMillis(); + LOG.log(result ? DEBUG : INFO, + () -> "Waiting finished after " + millis + " ms. Action " + (result ? "succeeded." : "timed out.")); + return result; } @@ -298,7 +379,7 @@ private static String[] getSystemPath() { private static class DotPrinter extends Thread { - public DotPrinter() { + private DotPrinter() { super("DotPrinter"); setDaemon(true); } diff --git a/nucleus/common/common-util/src/main/java/com/sun/enterprise/util/HostAndPort.java b/nucleus/common/common-util/src/main/java/com/sun/enterprise/util/HostAndPort.java index 0cbdf17cfeb..0b6775560b0 100644 --- a/nucleus/common/common-util/src/main/java/com/sun/enterprise/util/HostAndPort.java +++ b/nucleus/common/common-util/src/main/java/com/sun/enterprise/util/HostAndPort.java @@ -17,6 +17,7 @@ package com.sun.enterprise.util; +import java.net.InetSocketAddress; import java.util.Objects; /** @@ -65,6 +66,13 @@ public int getPort() { return port; } + /** + * @return new {@link InetSocketAddress} + */ + public InetSocketAddress toInetSocketAddress() { + return new InetSocketAddress(getHost(), getPort()); + } + @Override public int hashCode() { return Objects.hash(host, port, secure); @@ -87,6 +95,6 @@ public boolean equals(Object obj) { @Override public String toString() { - return host + ":" + port + (secure ? " (encrypted)" : " (unencrypted)"); + return host + ":" + port; } } diff --git a/nucleus/test-utils/src/main/java/org/glassfish/tests/utils/ServerUtils.java b/nucleus/test-utils/src/main/java/org/glassfish/tests/utils/ServerUtils.java index 0952a3474a3..a86e23cad25 100644 --- a/nucleus/test-utils/src/main/java/org/glassfish/tests/utils/ServerUtils.java +++ b/nucleus/test-utils/src/main/java/org/glassfish/tests/utils/ServerUtils.java @@ -89,7 +89,6 @@ public static int getFreePort(Set excluded) throws IllegalStateExceptio try (ServerSocket socket = new ServerSocket(0)) { final int port = socket.getLocalPort(); socket.setSoTimeout(1); - socket.setReuseAddress(true); if (excluded.contains(port) && counter >= 20) { throw new IllegalStateException("Cannot open random port, tried 20 times. Port " + port + " is excluded and we were not able to find another."); From cf25c6d0414f20e82f747f47d10a7674ad8a7d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mat=C4=9Bj=C4=8Dek?= Date: Wed, 26 Nov 2025 00:36:43 +0100 Subject: [PATCH 06/10] Use random ports rather than hard coded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Matějček --- .../appserv-tests/devtests/ejb/run_test.sh | 58 +++++++++++++++---- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/appserver/tests/appserv-tests/devtests/ejb/run_test.sh b/appserver/tests/appserv-tests/devtests/ejb/run_test.sh index 86011590507..3a4818473c6 100755 --- a/appserver/tests/appserv-tests/devtests/ejb/run_test.sh +++ b/appserver/tests/appserv-tests/devtests/ejb/run_test.sh @@ -15,6 +15,41 @@ # SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 # +get_12_free_ports() { + local ports=() + local port + local found + + for attempt in {1..1000}; do + [ ${#ports[@]} -ge 12 ] && break + + port=$((49152 + RANDOM % 16384)) + + # Skip if already in our list + found=false + for p in "${ports[@]}"; do + if [ "$p" -eq "$port" ]; then + found=true + break + fi + done + [ "$found" = true ] && continue + + # Check if port is free + if ! timeout 0.1 bash -c "/dev/null; then + ports+=("$port") + fi + done + + if [ ${#ports[@]} -eq 12 ]; then + echo "${ports[@]}" + return 0 + else + echo "ERROR: Only found ${#ports[@]} free ports" >&2 + return 1 + fi +} + test_run_ejb(){ rm -rf ${S1AS_HOME}/domains/domain1 @@ -22,17 +57,18 @@ test_run_ejb(){ echo "AS_ADMIN_PASSWORD=" > temppwd cat ${APS_HOME}/temppwd - ADMIN_PORT=45707 - JMS_PORT=45708 - JMX_PORT=45709 - ORB_PORT=45710 - SSL_PORT=45711 - INSTANCE_PORT=45712 - ALTERNATE_PORT=45713 - ORB_SSL_PORT=45714 - ORB_SSL_MUTUALAUTH_PORT=45715 - DB_PORT=45716 - DB_PORT_2=45717 + PORTS=($(get_12_free_ports)) + ADMIN_PORT=${PORTS[0]} + JMS_PORT=${PORTS[1]} + JMX_PORT=${PORTS[2]} + ORB_PORT=${PORTS[3]} + SSL_PORT=${PORTS[4]} + INSTANCE_PORT=${PORTS[5]} + ALTERNATE_PORT=${PORTS[6]} + ORB_SSL_PORT=${PORTS[7]} + ORB_SSL_MUTUALAUTH_PORT=${PORTS[8]} + DB_PORT=${PORTS[9]} + DB_PORT_2=${PORTS[10]} ${S1AS_HOME}/bin/asadmin \ --user anonymous \ From d615f2ed87a1f777630be537e55b4fb39a6fcce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mat=C4=9Bj=C4=8Dek?= Date: Wed, 26 Nov 2025 00:37:34 +0100 Subject: [PATCH 07/10] Retry binding to avoid zombies + postpone shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Matějček --- .../iiop/api/GlassFishORBHelper.java | 8 ++ .../iiop/impl/IIOPSSLSocketFactory.java | 85 ++++++++----------- 2 files changed, 42 insertions(+), 51 deletions(-) diff --git a/appserver/orb/orb-connector/src/main/java/org/glassfish/enterprise/iiop/api/GlassFishORBHelper.java b/appserver/orb/orb-connector/src/main/java/org/glassfish/enterprise/iiop/api/GlassFishORBHelper.java index 70f7a72139e..de499b33d67 100644 --- a/appserver/orb/orb-connector/src/main/java/org/glassfish/enterprise/iiop/api/GlassFishORBHelper.java +++ b/appserver/orb/orb-connector/src/main/java/org/glassfish/enterprise/iiop/api/GlassFishORBHelper.java @@ -97,6 +97,14 @@ private void onShutdown() { // Still, threads already working with the instance will have it unstable. final ORB destroyedOrb = orb; orb = null; + // FIXME: com.sun.corba.ee.impl.transport.AcceptorImpl.getAcceptedSocket(AcceptorImpl.java:127) + // can still be blocked in standalone thread, that would lead to its failure + // and cascade leading sockets open. Restart of the server could fail then. + try { + Thread.sleep(1000L); + } catch (InterruptedException e) { + // We don't want to interrupt here. + } destroyedOrb.destroy(); } } diff --git a/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java b/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java index 7c326aa2612..023e95b5283 100644 --- a/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java +++ b/appserver/orb/orb-iiop/src/main/java/org/glassfish/enterprise/iiop/impl/IIOPSSLSocketFactory.java @@ -23,8 +23,6 @@ import com.sun.corba.ee.spi.transport.ORBSocketFactory; import com.sun.enterprise.config.serverbeans.Config; import com.sun.enterprise.security.integration.AppClientSSL; -import com.sun.enterprise.universal.process.ProcessUtils; -import com.sun.enterprise.util.HostAndPort; import com.sun.logging.LogDomains; import java.io.IOException; @@ -35,7 +33,9 @@ import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Hashtable; import java.util.List; import java.util.Map; @@ -265,9 +265,11 @@ public ServerSocket createServerSocket(String type, InetSocketAddress inetSocket } else { serverSocket = new ServerSocket(); } - checkPort(inetSocketAddress); - serverSocket.bind(inetSocketAddress); - return serverSocket; + final Action action = () -> { + serverSocket.bind(inetSocketAddress); + return serverSocket; + }; + return repeat(action, Duration.ofSeconds(10L)); } /** @@ -349,60 +351,24 @@ private ServerSocket createSSLServerSocket(String type, InetSocketAddress inetSo String[] socketCiphers = ssf.getDefaultCipherSuites(); ciphers = mergeCiphers(socketCiphers, ssl3TlsCiphers, ssl2Ciphers); } - - String cs[] = null; - - if (LOG.isLoggable(Level.FINE)) { - cs = ssf.getSupportedCipherSuites(); - for (String element : cs) { - LOG.log(Level.FINE, "Cipher Suite: " + element); - } - } - - ServerSocket ss = null; - try { - // bugfix for 6349541 - // specify the ip address to bind to, 50 is the default used - // by the ssf implementation when only the port is specified - checkPort(inetSocketAddress); - ss = ssf.createServerSocket(port, BACKLOG, inetSocketAddress.getAddress()); - if (ciphers != null) { - ((SSLServerSocket) ss).setEnabledCipherSuites(ciphers); - } - } catch (IOException e) { - LOG.log(Level.SEVERE, "createServerSocket failed", new Object[] {type, port}); - LOG.log(Level.SEVERE, "", e); - throw e; + LOG.log(Level.FINE, () -> "Supported cipher Suites: " + Arrays.toString(ssf.getSupportedCipherSuites())); + final Action action = () -> ssf.createServerSocket(port, BACKLOG, inetSocketAddress.getAddress()); + final ServerSocket ss = repeat(action, Duration.ofSeconds(10L)); + if (ciphers != null) { + ((SSLServerSocket) ss).setEnabledCipherSuites(ciphers); } - try { if (type.equals(SSL_MUTUALAUTH)) { LOG.log(Level.FINE, "Setting Mutual auth"); ((SSLServerSocket) ss).setNeedClientAuth(true); } } catch (Exception e) { - LOG.log(Level.SEVERE, "Setting Mutual auth failed.", e); - throw new IOException(e.getMessage()); - } - if (LOG.isLoggable(Level.FINE)) { - LOG.log(Level.FINE, "Created server socket:" + ss); + throw new IOException(e.getMessage(), e); } + LOG.log(Level.FINE, () -> "Created server socket: " + ss); return ss; } - /** FIXME Temporary hack until we find out which part is leaking. */ - private static void checkPort(InetSocketAddress address) { - int port = address.getPort(); - if (port < 1) { - return; - } - final HostAndPort endpoint = new HostAndPort(address.getHostString(), port, false); - if (!ProcessUtils.isListening(endpoint)) { - return; - } - ProcessUtils.waitWhileListening(endpoint, Duration.ofSeconds(10L), false); - } - /** * Create an SSL socket at the specified host and port. * @param host @@ -437,9 +403,7 @@ private Socket createSSLSocket(String host, int port) throws IOException { LOG.log(Level.FINE, "createSSLSocket failed.", new Object[] {host, port}); LOG.log(Level.FINE, "", e); } - IOException e2 = new IOException("Error opening SSL socket to host=" + host + " port=" + port); - e2.initCause(e); - throw e2; + throw new IOException("Error opening SSL socket to host=" + host + " port=" + port, e); } return socket; } @@ -563,6 +527,20 @@ private boolean isValidProtocolCipher(CipherInfo cipherInfo, } + private static T repeat(Action action, Duration timeout) throws IOException { + final Instant deadline = Instant.now().plus(timeout); + while (true) { + try { + return action.get(); + } catch (Exception e) { + if (Instant.now().isAfter(deadline)) { + throw e; + } + } + } + } + + class SSLInfo { private final SSLContext ctx; private String[] ssl3TlsCiphers = null; @@ -586,4 +564,9 @@ String[] getSsl2Ciphers() { return ssl2Ciphers; } } + + @FunctionalInterface + private interface Action { + T get() throws IOException; + } } From 5234fa520aa30c08a5f5783659ed9536314feb68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mat=C4=9Bj=C4=8Dek?= Date: Wed, 26 Nov 2025 01:43:58 +0100 Subject: [PATCH 08/10] Better random port generator written in java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Matějček --- .../appserv-tests/devtests/ejb/ports.java | 76 +++++++++++++++++++ .../appserv-tests/devtests/ejb/run_test.sh | 33 +------- 2 files changed, 77 insertions(+), 32 deletions(-) create mode 100644 appserver/tests/appserv-tests/devtests/ejb/ports.java diff --git a/appserver/tests/appserv-tests/devtests/ejb/ports.java b/appserver/tests/appserv-tests/devtests/ejb/ports.java new file mode 100644 index 00000000000..26e14c0f14a --- /dev/null +++ b/appserver/tests/appserv-tests/devtests/ejb/ports.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2025 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0, which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the + * Eclipse Public License v. 2.0 are satisfied: GNU General Public License, + * version 2 with the GNU Classpath Exception, which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + */ + +import java.io.File; +import java.io.IOException; +import java.net.ServerSocket; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.Map.Entry; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public class ports { + + public static void main(String... args) throws Exception { + System.out.println(getFreePorts(12).stream().collect(Collectors.joining(" "))); + } + + /** + * Tries to allocate a free local ports, avoids duplicates. + * + * @param count count of free ports to find. + * @return a modifiable queue of free local port numbers. + * @throws IllegalStateException if it fails for 20 times + */ + private static Queue getFreePorts(int count) throws IllegalStateException { + final ArrayDeque generatedPorts = new ArrayDeque<>(count); + final Set excludedPorts = new HashSet<>(); + for (int i = 0; i < count; i++) { + String port = getFreePort(excludedPorts); + generatedPorts.add(port); + // Avoid duplicates + excludedPorts.add(port); + } + return generatedPorts; + } + + private static String getFreePort(Set excluded) throws IllegalStateException { + int counter = 0; + while (true) { + counter++; + try (ServerSocket socket = new ServerSocket(0)) { + final int port = socket.getLocalPort(); + socket.setSoTimeout(1); + if (excluded.contains(port) && counter >= 20) { + throw new IllegalStateException("Cannot open random port, tried 20 times. Port " + port + + " is excluded and we were not able to find another."); + } + return Integer.toString(port); + } catch (IOException e) { + if (counter >= 20) { + throw new IllegalStateException("Cannot open random port, tried 20 times.", e); + } + } + } + } +} diff --git a/appserver/tests/appserv-tests/devtests/ejb/run_test.sh b/appserver/tests/appserv-tests/devtests/ejb/run_test.sh index 3a4818473c6..d9c1fb88aac 100755 --- a/appserver/tests/appserv-tests/devtests/ejb/run_test.sh +++ b/appserver/tests/appserv-tests/devtests/ejb/run_test.sh @@ -16,38 +16,7 @@ # get_12_free_ports() { - local ports=() - local port - local found - - for attempt in {1..1000}; do - [ ${#ports[@]} -ge 12 ] && break - - port=$((49152 + RANDOM % 16384)) - - # Skip if already in our list - found=false - for p in "${ports[@]}"; do - if [ "$p" -eq "$port" ]; then - found=true - break - fi - done - [ "$found" = true ] && continue - - # Check if port is free - if ! timeout 0.1 bash -c "/dev/null; then - ports+=("$port") - fi - done - - if [ ${#ports[@]} -eq 12 ]; then - echo "${ports[@]}" - return 0 - else - echo "ERROR: Only found ${#ports[@]} free ports" >&2 - return 1 - fi + ${JAVA_HOME}/bin/java ${APS_HOME}/devtests/ejb/ports.java } test_run_ejb(){ From f103aa2c032d0c155c3fbb32da17354fe6ae7116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mat=C4=9Bj=C4=8Dek?= Date: Wed, 26 Nov 2025 15:21:14 +0100 Subject: [PATCH 09/10] Fixed reporting when the start failed. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Matějček --- .../servermgmt/cli/ServerLifeSignChecker.java | 28 +++++++++---------- .../servermgmt/cli/StopDomainCommand.java | 4 +++ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignChecker.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignChecker.java index b17f3afe359..110db52f6fc 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignChecker.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/ServerLifeSignChecker.java @@ -73,23 +73,23 @@ public ServerLifeSigns watchStartup(GlassFishProcess process, Duration timeout) if (wasTimeout) { return createTimeoutReport(signs); } - if (!process.isAlive()) { - signs.error = true; - signs.suggestion = getSuggestions(); - final Integer exitCode = process.exitCode(); - if (exitCode == null) { - signs.summary = "The process died."; + if (process.isAlive()) { + signs.summary = "Successfully started the " + checks.getServerTitleAndName() + "."; + return signs; + } + signs.error = true; + signs.suggestion = getSuggestions(); + final Integer exitCode = process.exitCode(); + if (exitCode == null) { + signs.summary = "The process died."; + } else { + signs.summary = "The startup command return code was " + exitCode + " which means that the start "; + if (exitCode == 0) { + signs.summary += "succeded, however later the process stopped for some reason."; } else { - signs.summary = "The startup command return code was " + exitCode + " which means that the start "; - if (exitCode == 0) { - signs.summary += "succeded, however later the process stopped for some reason."; - } else { - signs.summary += "failed with exit code " + exitCode + "."; - } + signs.summary += "failed."; } - return signs; } - signs.summary = "Successfully started the " + checks.getServerTitleAndName() + "."; return signs; } diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StopDomainCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StopDomainCommand.java index 883ef2c7c03..a1a95509f33 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StopDomainCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StopDomainCommand.java @@ -152,6 +152,8 @@ protected int dasNotRunning() throws CommandException { if (isLocal()) { try { File prevPid = getServerDirs().getLastPidFile(); + LOG.log(Level.WARNING, "The domain admin port could not be reached." + + " We will try to kill the process with PID " + prevPid); ProcessUtils.kill(prevPid, getStopTimeout(), !programOpts.isTerse()); } catch (KillNotPossibleException e) { throw new CommandException(e.getMessage(), e); @@ -195,6 +197,8 @@ private void localShutdown(Long pid, Duration stopTimeout, boolean printDots) th if (kill && isLocal()) { try { File prevPid = getServerDirs().getLastPidFile(); + LOG.log(Level.WARNING, "The stop-domain command timed out." + + " We will try to kill the process with PID " + prevPid); ProcessUtils.kill(prevPid, stopTimeout, printDots); return; } catch (Exception ex) { From 01101fb1da4a990141fc658558f02685ae247d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mat=C4=9Bj=C4=8Dek?= Date: Wed, 26 Nov 2025 15:25:24 +0100 Subject: [PATCH 10/10] Debug port can be still unbindable after previous JVM died MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - We have issues on Windows on GHA with this. - Maybe local restarts should be done as stop-start on Windows? Respawning is not recommended and on windows it seems really risky Signed-off-by: David Matějček --- .../main/itest/tools/asadmin/Asadmin.java | 2 +- .../main/admin/test/AsadminLoggingITest.java | 17 ++++++++++++----- .../enterprise/admin/launcher/GFLauncher.java | 2 +- .../servermgmt/cli/LocalServerCommand.java | 2 +- .../admin/servermgmt/cli/StartServerHelper.java | 8 ++------ 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/appserver/itest-tools/src/main/java/org/glassfish/main/itest/tools/asadmin/Asadmin.java b/appserver/itest-tools/src/main/java/org/glassfish/main/itest/tools/asadmin/Asadmin.java index c0e0dfb5ff1..296f01fc801 100644 --- a/appserver/itest-tools/src/main/java/org/glassfish/main/itest/tools/asadmin/Asadmin.java +++ b/appserver/itest-tools/src/main/java/org/glassfish/main/itest/tools/asadmin/Asadmin.java @@ -242,7 +242,7 @@ private File getPasswordFile() { */ private AsadminResult exec(final Integer timeout, final boolean detachedAndTerse, final String... args) { final List parameters = Arrays.asList(args); - LOG.log(TRACE, "exec(timeout={0}, detached={1}, args={2})", timeout, detachedAndTerse, parameters); + LOG.log(INFO, "exec(timeout={0}, detached={1}, args={2})", timeout, detachedAndTerse, parameters); final List command = new ArrayList<>(); if (asadmin.getName().endsWith(".java")) { command.add(JAVA_EXECUTABLE); diff --git a/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/AsadminLoggingITest.java b/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/AsadminLoggingITest.java index 91dedbbbbe0..e6046a29aa1 100644 --- a/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/AsadminLoggingITest.java +++ b/appserver/tests/admin/tests/src/test/java/org/glassfish/main/admin/test/AsadminLoggingITest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2024 Contributors to the Eclipse Foundation. + * Copyright (c) 2022, 2025 Contributors to the Eclipse Foundation. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v. 2.0, which is available at @@ -16,6 +16,8 @@ package org.glassfish.main.admin.test; +import com.sun.enterprise.util.OS; + import java.io.File; import java.io.FileReader; import java.io.LineNumberReader; @@ -69,11 +71,16 @@ public class AsadminLoggingITest { private static final Asadmin ASADMIN = GlassFishTestEnvironment.getAsadmin(); + /** Fill up the server log. */ @BeforeAll - public static void fillUpServerLog() { - // Fill up the server log. - AsadminResult result = ASADMIN.exec("restart-domain", "--timeout", "60"); - assertThat(result, asadminOK()); + public static void fillUpServerLog() throws Exception { + if (OS.isWindowsForSure()) { + // For some reason windows can collide on debug port. + assertThat(ASADMIN.exec("stop-domain"), asadminOK()); + assertThat(ASADMIN.exec("start-domain"), asadminOK()); + } else { + assertThat(ASADMIN.exec("restart-domain"), asadminOK()); + } } @Test diff --git a/nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java b/nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java index 34d271d4f4f..09158c40495 100644 --- a/nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java +++ b/nucleus/admin/launcher/src/main/java/com/sun/enterprise/admin/launcher/GFLauncher.java @@ -211,6 +211,7 @@ public final void launch() throws GFLauncherException { LOG.log(INFO, () -> "Debugging will be available on port " + debugPort + "."); } } + logCommandLine(); try { startTime = System.currentTimeMillis(); if (isFakeLaunch()) { @@ -284,7 +285,6 @@ public void setup() throws GFLauncherException, MiniXmlParserException { setClasspath(); initCommandLine(); setJvmOptions(); - logCommandLine(); // if no element, we need to upgrade this domain needsAutoUpgrade = !domainXML.hasNetworkConfig(); diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java index 527502b6a7e..8bf9b63a26d 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/LocalServerCommand.java @@ -404,7 +404,7 @@ protected final String waitForStart(final Long oldPid, final ServerLifeSignCheck if (startTimeout != null && startTimeout.isNegative()) { throw new CommandException(reportPidFileIssue(pidFile)); } - final Long pid = loadPid(getServerDirs().getPidFile()); + final Long pid = pidFile.isFile() ? loadPid(pidFile) : null; if (pid == null) { throw new CommandException(reportPidFileIssue(pidFile)); } diff --git a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java index ea6aac70ffa..7cbede62037 100644 --- a/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java +++ b/nucleus/admin/server-mgmt/src/main/java/com/sun/enterprise/admin/servermgmt/cli/StartServerHelper.java @@ -277,14 +277,10 @@ public static List parseCustomEndpoints(String customEndpoints) thr * So we have to wait a bit. */ private static void checkFreeDebugPort(Integer debugPort, Duration timeout, boolean terse) { - if (debugPort == null) { + if (debugPort == null || NetUtils.isPortFree(debugPort)) { return; } - final HostAndPort debugEndpoint = new HostAndPort("localhost", debugPort, false); - if (!ProcessUtils.isListening(debugEndpoint)) { - return; - } - ProcessUtils.waitWhileListening(debugEndpoint, timeout, !terse); + ProcessUtils.waitFor(() -> NetUtils.isPortFree(debugPort), Duration.ofSeconds(10L), terse); } private static void checkFreeAdminPorts(List endpoints) throws GFLauncherException {