Allow cookies to be sent with HTTP replication requests

osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/common/ReplicationCookie.java

@@ -0,0 +1,124 @@
+// This software is released into the Public Domain.  See copying.txt for details.
+package org.openstreetmap.osmosis.replication.common;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+import org.openstreetmap.osmosis.core.OsmosisConstants;
+import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
+
+/**
+ * Cookie to be sent with all HTTP requests. The cookie is read from a file.
+ *
+ * @author Michael Reichert
+ */
+public class ReplicationCookie {
+	private static final String COOKIE_FILE_NAME = "cookie.txt";
+
+	private Path directory;
+	private URL cookieStatusAPI;
+	private String data;
+
+	/**
+	 * Create an invalid dummy cookie.
+	 */
+	public ReplicationCookie() {
+		directory = null;
+		cookieStatusAPI = null;
+		data = null;
+	}
+
+	/**
+	 * Creates an empty cookie.
+	 *
+	 * @param cookieDirectory directory to read the cookie.txt from
+	 */
+	public ReplicationCookie(Path cookieDirectory, URL cookieStatusApiUrl) {
+		directory = cookieDirectory;
+		cookieStatusAPI = cookieStatusApiUrl;
+		data = "";
+	}
+
+	/**
+	 * Check if this cookie is not empty and used.
+	 * 
+	 * @return False if it has not been set.
+	 */
+	public boolean valid() {
+		return data != null && !data.isEmpty();
+	}
+
+	/**
+	 * Get the string representation of the cookie to be set as HTTP header.
+	 * 
+	 * @return string representation
+	 */
+	public String toString() {
+		return data;
+	}
+
+	/**
+	 * Read the cookie from a file name cookie.txt in the working directory.
+	 */
+	public void read() {
+		Path cookieFilePath = directory.resolve(Paths.get(COOKIE_FILE_NAME));
+		try {
+			List<String> lines = Files.readAllLines(cookieFilePath, Charset.forName("US-ASCII"));
+			if (lines.size() == 1) {
+				data = lines.get(0);
+			} else {
+				throw new OsmosisRuntimeException("The cookie file " + cookieFilePath.toString()
+				+ " must contain exactly one line.");
+			}
+		} catch (IOException e) {
+			throw new OsmosisRuntimeException("Failed to read the cookie file " + cookieFilePath.toString());
+		}
+	}
+
+	/**
+	 * Throw a OsmosisRuntimeException if the cookie isn't accepted by the server any more.
+	 *
+	 * @throws OsmosisRuntimeException
+	 */
+	public void throw_if_expired() {
+		if (!accepted()) {
+			throw new OsmosisRuntimeException("Your cookie is not valid anymore.");
+		}
+	}
+
+	/**
+	 * Check if the cookie is still accepted by the server
+	 *
+	 * @return acceptance
+	 *
+	 * @throws OsmosisRuntimeException for empty cookies and IOExceptions
+	 */
+	public boolean accepted() {
+		if (!valid()) {
+			throw new OsmosisRuntimeException("Cannot check if the cookie is expired because it is empty.");
+		}
+		if (cookieStatusAPI == null) {
+			return false;
+		}
+		HttpURLConnection connection;
+		try {
+			connection = (HttpURLConnection) cookieStatusAPI.openConnection();
+			connection.setReadTimeout(15 * 60 * 1000); // timeout 15 minutes
+			connection.setConnectTimeout(15 * 60 * 1000); // timeout 15 minutes
+			connection.setRequestProperty("User-Agent", "Osmosis/" + OsmosisConstants.VERSION);
+			connection.setRequestProperty("Cookie", data);
+			// A HTTP HEAD request is sufficient, we don't have to parse the JSON.
+			connection.setRequestMethod("HEAD");
+			connection.connect();
+			return connection.getResponseCode() == HttpURLConnection.HTTP_OK;
+		} catch (IOException e) {
+			throw new OsmosisRuntimeException("Failed to check if the cookie is still valid.");
+		}
+	}
+}

osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/common/ServerStateReader.java

@@ -38,26 +38,61 @@ public ServerStateReader() {
 	/**
 	 * Retrieves the latest state from the server.
 	 * 
+	 * No cookie will be sent with the HTTP request.
+	 *
 	 * @param baseUrl
 	 *            The url of the directory containing change files.
 	 * @return The state.
 	 */
 	public ReplicationState getServerState(URL baseUrl) {
-		return getServerState(baseUrl, SERVER_STATE_FILE);
+		return getServerState(baseUrl, SERVER_STATE_FILE, new ReplicationCookie());
+	}
+
+
+	/**
+	 * Retrieves the latest state from the server.
+	 *
+	 * @param baseUrl
+	 *            The url of the directory containing change files.
+	 * @param cookie
+	 *            Cookie to send with each HTTP request.
+	 * @return The state.
+	 */
+	public ReplicationState getServerState(URL baseUrl, ReplicationCookie cookie) {
+		return getServerState(baseUrl, SERVER_STATE_FILE, cookie);
 	}
 
 
 	/**
 	 * Retrieves the specified state from the server.
 	 * 
+	 * No cookie will be sent with the HTTP request.
+	 *
 	 * @param baseUrl
 	 *            The url of the directory containing change files.
 	 * @param sequenceNumber
 	 *            The sequence number of the state to be retrieved from the server.
 	 * @return The state.
 	 */
 	public ReplicationState getServerState(URL baseUrl, long sequenceNumber) {
-		return getServerState(baseUrl, sequenceFormatter.getFormattedName(sequenceNumber, SEQUENCE_STATE_FILE_SUFFIX));
+		return getServerState(baseUrl, sequenceNumber, new ReplicationCookie());
+	}
+
+
+	/**
+	 * Retrieves the specified state from the server.
+	 *
+	 * @param baseUrl
+	 *            The url of the directory containing change files.
+	 * @param sequenceNumber
+	 *            The sequence number of the state to be retrieved from the server.
+	 * @param cookie
+	 *            Cookie to send with each HTTP request.
+	 * @return The state.
+	 */
+	public ReplicationState getServerState(URL baseUrl, long sequenceNumber, ReplicationCookie cookie) {
+		return getServerState(baseUrl, sequenceFormatter.getFormattedName(sequenceNumber, SEQUENCE_STATE_FILE_SUFFIX),
+				cookie);
 	}
 
 
@@ -68,9 +103,11 @@ public ReplicationState getServerState(URL baseUrl, long sequenceNumber) {
 	 *            The url of the directory containing change files.
 	 * @param stateFile
 	 *            The state file to be retrieved.
+	 * @param cookie
+	 *            Cookie to send with each HTTP request.
 	 * @return The state.
 	 */
-	private ReplicationState getServerState(URL baseUrl, String stateFile) {
+	private ReplicationState getServerState(URL baseUrl, String stateFile, ReplicationCookie cookie) {
 		URL stateUrl;
 
 		try {
@@ -88,6 +125,9 @@ private ReplicationState getServerState(URL baseUrl, String stateFile) {
 			connection.setReadTimeout(15 * 60 * 1000); // timeout 15 minutes
 			connection.setConnectTimeout(15 * 60 * 1000); // timeout 15 minutes
 			connection.setRequestProperty("User-Agent", "Osmosis/" + OsmosisConstants.VERSION);
+			if (cookie.valid()) {
+				connection.setRequestProperty("Cookie", cookie.toString());
+			}
 			try (BufferedReader reader  = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
 				stateProperties = new Properties();
 				stateProperties.load(reader);

osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/BaseReplicationDownloader.java

@@ -19,6 +19,7 @@
 import org.openstreetmap.osmosis.core.task.common.RunnableTask;
 import org.openstreetmap.osmosis.core.util.FileBasedLock;
 import org.openstreetmap.osmosis.core.util.PropertiesPersister;
+import org.openstreetmap.osmosis.replication.common.ReplicationCookie;
 import org.openstreetmap.osmosis.replication.common.ReplicationSequenceFormatter;
 import org.openstreetmap.osmosis.replication.common.ReplicationState;
 import org.openstreetmap.osmosis.replication.common.ServerStateReader;
@@ -45,6 +46,13 @@ public abstract class BaseReplicationDownloader implements RunnableTask {
 	private File workingDirectory;
 	private ReplicationSequenceFormatter sequenceFormatter;
 	private ServerStateReader serverStateReader;
+
+	/**
+	 * configuration of the downloader
+	 */
+	protected ReplicationDownloaderConfiguration configuration;
+
+	private ReplicationCookie cookie;
 
 
 	/**
@@ -58,6 +66,12 @@ public BaseReplicationDownloader(File workingDirectory) {
 
 		sequenceFormatter = new ReplicationSequenceFormatter(9, 3);
 		serverStateReader = new ServerStateReader();
+
+		configuration = new ReplicationDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE));
+		cookie = new ReplicationCookie(workingDirectory.toPath(), configuration.getCookieStatusAPI());
+		if (configuration.getAttachCookie()) {
+			cookie.read();
+		}
 	}
 
 
@@ -98,7 +112,10 @@ private File downloadReplicationFile(String fileName, URL baseUrl) {
 			connection.setReadTimeout(15 * 60 * 1000); // timeout 15 minutes
 			connection.setConnectTimeout(15 * 60 * 1000); // timeout 15 minutes
 			connection.setRequestProperty("User-Agent", "Osmosis/" + OsmosisConstants.VERSION);
-
+			if (cookie.valid()) {
+				connection.setRequestProperty("Cookie", cookie.toString());
+			}
+
 			try (BufferedInputStream source = new BufferedInputStream(connection.getInputStream(), 65536)) {
 				// Create a temporary file to write the data to.
 				outputFile = File.createTempFile("change", null);
@@ -146,16 +163,13 @@ private void processReplicationFile(File replicationFile, ReplicationState repli
 	 * limit the maximum timestamp further if needed. A sub-class may never increase the maximum
 	 * timestamp beyond that calculated by this method.
 	 * 
-	 * @param configuration
-	 *            The configuration.
 	 * @param serverTimestamp
 	 *            The timestamp of the latest data on the server.
 	 * @param localTimestamp
 	 *            The timestamp of the most recently downloaded data.
 	 * @return The maximum timestamp for this invocation.
 	 */
-	protected Date calculateMaximumTimestamp(ReplicationDownloaderConfiguration configuration, Date serverTimestamp,
-			Date localTimestamp) {
+	protected Date calculateMaximumTimestamp(Date serverTimestamp, Date localTimestamp) {
 		Date maximumTimestamp;
 
 		maximumTimestamp = serverTimestamp;
@@ -174,8 +188,7 @@ protected Date calculateMaximumTimestamp(ReplicationDownloaderConfiguration conf
 	}
 
 
-	private ReplicationState download(ReplicationDownloaderConfiguration configuration, ReplicationState serverState,
-			ReplicationState initialLocalState) {
+	private ReplicationState download(ReplicationState serverState, ReplicationState initialLocalState) {
 		URL baseUrl;
 		ReplicationState localState;
 		Date maximumDownloadTimestamp;
@@ -187,7 +200,7 @@ private ReplicationState download(ReplicationDownloaderConfiguration configurati
 
 		// Determine the maximum timestamp that can be downloaded.
 		maximumDownloadTimestamp =
-			calculateMaximumTimestamp(configuration, serverState.getTimestamp(), localState.getTimestamp());
+			calculateMaximumTimestamp(serverState.getTimestamp(), localState.getTimestamp());
 		LOG.fine("The maximum timestamp to be downloaded is " + maximumDownloadTimestamp + ".");
 
 		// Download all files and send their contents to the sink.
@@ -210,7 +223,7 @@ private ReplicationState download(ReplicationDownloaderConfiguration configurati
 			LOG.finer("Processing replication sequence " + sequenceNumber + ".");
 
 			// Get the state associated with the next file.
-			fileReplicationState = serverStateReader.getServerState(baseUrl, sequenceNumber);
+			fileReplicationState = serverStateReader.getServerState(baseUrl, sequenceNumber, cookie);
 
 			// Ensure that the next state is within the allowable timestamp
 			// range. We must stop if the next data takes us beyond the maximum
@@ -244,17 +257,13 @@ private ReplicationState download(ReplicationDownloaderConfiguration configurati
 
 	private void runImpl() {
 		try {
-			ReplicationDownloaderConfiguration configuration;
 			ReplicationState serverState;
 			ReplicationState localState;
 			PropertiesPersister localStatePersistor;
 
-			// Instantiate utility objects.
-			configuration = new ReplicationDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE));
-
 			// Obtain the server state.
 			LOG.fine("Reading current server state.");
-			serverState = serverStateReader.getServerState(configuration.getBaseUrl());
+			serverState = serverStateReader.getServerState(configuration.getBaseUrl(), cookie);
 
 			// Build the local state persister which is used for both loading and storing local state.
 			localStatePersistor = new PropertiesPersister(new File(workingDirectory, LOCAL_STATE_FILE));
@@ -268,7 +277,7 @@ private void runImpl() {
 				localState = new ReplicationState(localStatePersistor.loadMap());
 
 				// Download and process the replication files.
-				localState = download(configuration, serverState, localState);
+				localState = download(serverState, localState);
 
 			} else {
 				localState = serverState;

osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/ReplicationFileMerger.java

@@ -13,7 +13,6 @@
 import org.openstreetmap.osmosis.replication.common.FileReplicationStore;
 import org.openstreetmap.osmosis.replication.common.ReplicationState;
 import org.openstreetmap.osmosis.replication.common.ReplicationStore;
-import org.openstreetmap.osmosis.replication.v0_6.impl.ReplicationDownloaderConfiguration;
 import org.openstreetmap.osmosis.replication.v0_6.impl.ReplicationFileMergerConfiguration;
 import org.openstreetmap.osmosis.xml.v0_6.XmlChangeReader;
 import org.openstreetmap.osmosis.xml.v0_6.XmlChangeWriter;
@@ -69,16 +68,15 @@ private Date alignDateToIntervalBoundary(Date requestedDate, long intervalLength
 	 * {@inheritDoc}
 	 */
 	@Override
-	protected Date calculateMaximumTimestamp(ReplicationDownloaderConfiguration configuration, Date serverTimestamp,
-			Date localTimestamp) {
+	protected Date calculateMaximumTimestamp(Date serverTimestamp, Date localTimestamp) {
 		Date maximumTimestamp;
 		long intervalLength;
 
 		// Read the current persisted state.
 		currentDataState = replicationStore.getCurrentState();
 
 		// Get the default maximum timestamp according to base calculations.
-		maximumTimestamp = super.calculateMaximumTimestamp(configuration, serverTimestamp, localTimestamp);
+		maximumTimestamp = super.calculateMaximumTimestamp(serverTimestamp, localTimestamp);
 
 		// Align the maximum timestamp to an interval boundary.
 		intervalLength = getConfiguration().getIntervalLength();

osmosis-replication/src/main/java/org/openstreetmap/osmosis/replication/v0_6/ReplicationLagReader.java

@@ -9,6 +9,7 @@
 import org.openstreetmap.osmosis.core.task.common.RunnableTask;
 import org.openstreetmap.osmosis.core.util.FileBasedLock;
 import org.openstreetmap.osmosis.core.util.PropertiesPersister;
+import org.openstreetmap.osmosis.replication.common.ReplicationCookie;
 import org.openstreetmap.osmosis.replication.common.ReplicationState;
 import org.openstreetmap.osmosis.replication.common.ServerStateReader;
 import org.openstreetmap.osmosis.replication.v0_6.impl.ReplicationDownloaderConfiguration;
@@ -59,10 +60,20 @@ private void getLag() {
 
 		// Instantiate utility objects.
 		configuration = new ReplicationDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE));
+
+		// read cookie if necessary
+		ReplicationCookie cookie = new ReplicationCookie(workingDirectory.toPath(),
+				configuration.getCookieStatusAPI());
+		if (configuration.getAttachCookie()) {
+			cookie.read();
+			// Throw an exception if the cookie is expired. Fail with a understandable error now because further
+			// requests will fail.
+			cookie.throw_if_expired();
+		}
 
 		// Obtain the server state.
 		LOG.fine("Reading current server state.");
-		serverState = serverStateReader.getServerState(configuration.getBaseUrl());
+		serverState = serverStateReader.getServerState(configuration.getBaseUrl(), cookie);
 
 		// Build the local state persister which is used for both loading and storing local state.
 		localStatePersistor = new PropertiesPersister(new File(workingDirectory, LOCAL_STATE_FILE));