Decouple the crawler service for use in other frameworks

gwtp-core/gwtp-crawler-guice/pom.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.gwtplatform</groupId>
+        <artifactId>gwtp-core</artifactId>
+        <version>1.6-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>gwtp-crawler-guice</artifactId>
+    <name>GWTP Crawler for Guice</name>
+
+    <build>
+        <resources>
+            <!-- Bundle sources with the jar, so they are visible to GWT's compiler -->
+            <resource>
+                <directory>src/main/java</directory>
+                <includes>
+                    <include>**/*.java</include>
+                </includes>
+            </resource>
+            <!-- Bundle module descriptor with the jar, so it is visible to GWT's compiler -->
+            <resource>
+                <directory>src/main/resources</directory>
+                <includes>
+                    <include>**/*.gwt.xml</include>
+                </includes>
+            </resource>
+        </resources>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>gwtp-crawler</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>javax.servlet</groupId>
+            <artifactId>servlet-api</artifactId>
+        </dependency>
+
+        <!-- DI dependencies -->
+        <dependency>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+        </dependency>
+    </dependencies>
+</project>

gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/ServiceKey.java → gwtp-core/gwtp-crawler-guice/src/main/java/com/gwtplatform/crawler/server/ServiceKey.java

@@ -14,13 +14,14 @@
  * the License.
  */
 
-package com.gwtplatform.crawlerservice.server;
+package com.gwtplatform.crawler.server;
+
+import com.google.inject.BindingAnnotation;
+import com.gwtplatform.crawler.server.guice.service.CrawlServiceServlet;
 
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;
 
-import com.google.inject.BindingAnnotation;
-
 import static java.lang.annotation.ElementType.FIELD;
 import static java.lang.annotation.ElementType.METHOD;
 import static java.lang.annotation.ElementType.PARAMETER;
@@ -31,10 +32,11 @@
  * {@link CrawlServiceServlet}. For example:
  * <pre>bindConstant().annotatedWith(ServiceKey.class).to("123456");
  * </pre>
+ * @deprecated Please use {@link com.gwtplatform.crawler.server.guice.ServiceKey} instead.
  */
-
 @BindingAnnotation
 @Target({FIELD, PARAMETER, METHOD})
 @Retention(RUNTIME)
+@Deprecated
 public @interface ServiceKey {
 }

gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/ServiceKey.java → gwtp-core/gwtp-crawler-guice/src/main/java/com/gwtplatform/crawler/server/guice/ServiceKey.java

@@ -14,12 +14,13 @@
  * the License.
  */
 
-package com.gwtplatform.crawler.server;
+package com.gwtplatform.crawler.server.guice;
 
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;
 
 import com.google.inject.BindingAnnotation;
+import com.gwtplatform.crawler.server.guice.service.CrawlServiceServlet;
 
 import static java.lang.annotation.ElementType.FIELD;
 import static java.lang.annotation.ElementType.METHOD;
@@ -32,7 +33,6 @@
  * <pre>bindConstant().annotatedWith(ServiceKey.class).to("123456");
  * </pre>
  */
-
 @BindingAnnotation
 @Target({FIELD, PARAMETER, METHOD})
 @Retention(RUNTIME)

gwtp-core/gwtp-crawler-guice/src/main/java/com/gwtplatform/crawler/server/guice/filter/GuiceCrawlFilter.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server.guice.filter;
+
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import com.gwtplatform.crawler.server.CrawlFilter;
+import com.gwtplatform.crawler.server.guice.ServiceKey;
+
+/**
+ * Guice implementation for the {@link CrawlFilter}.
+ * @author Ben Dol
+ */
+@Singleton
+public final class GuiceCrawlFilter extends CrawlFilter {
+
+    @Inject
+    GuiceCrawlFilter(@ServiceUrl String serviceUrl,
+                     @ServiceKey String key,
+                     Logger log) {
+        super(serviceUrl, key, log);
+    }
+}

gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/ServiceUrl.java → gwtp-core/gwtp-crawler-guice/src/main/java/com/gwtplatform/crawler/server/guice/filter/ServiceUrl.java

@@ -14,7 +14,7 @@
  * the License.
  */
 
-package com.gwtplatform.crawler.server;
+package com.gwtplatform.crawler.server.guice.filter;
 
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;

gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/CachedPageTimeoutSec.java → gwtp-core/gwtp-crawler-guice/src/main/java/com/gwtplatform/crawler/server/guice/service/CachedPageTimeoutSec.java

@@ -14,7 +14,7 @@
  * the License.
  */
 
-package com.gwtplatform.crawlerservice.server;
+package com.gwtplatform.crawler.server.guice.service;
 
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;

gwtp-core/gwtp-crawler-guice/src/main/java/com/gwtplatform/crawler/server/guice/service/CrawlServiceServlet.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server.guice.service;
+
+import java.util.logging.Logger;
+
+import javax.inject.Provider;
+import javax.inject.Singleton;
+
+import com.gargoylesoftware.htmlunit.WebClient;
+import com.google.inject.Inject;
+import com.gwtplatform.crawler.server.AbstractCrawlServiceServlet;
+import com.gwtplatform.crawler.server.CrawlCacheService;
+import com.gwtplatform.crawler.server.guice.ServiceKey;
+
+/**
+ * Guice Crawl Service Servlet.
+ * @author Ben Dol
+ */
+@Singleton
+public class CrawlServiceServlet extends AbstractCrawlServiceServlet {
+
+    @Inject(optional = true)
+    @HtmlUnitTimeoutMillis
+    private long timeoutMillis = 5000;
+
+    @Inject(optional = true)
+    @CachedPageTimeoutSec
+    private long cachedPageTimeoutSec = 15 * 60;
+
+    private final Provider<WebClient> webClientProvider;
+
+    @Inject
+    protected CrawlServiceServlet(
+            Provider<WebClient> webClientProvider,
+            Logger log,
+            CrawlCacheService crawlCacheService,
+            @ServiceKey String key) {
+        super(log, key, crawlCacheService);
+
+        this.webClientProvider = webClientProvider;
+    }
+
+    @Override
+    protected WebClient getWebClient() {
+        return webClientProvider.get();
+    }
+
+    @Override
+    public long getCachedPageTimeoutSec() {
+        return cachedPageTimeoutSec;
+    }
+
+    @Override
+    public long getTimeoutMillis() {
+        return timeoutMillis;
+    }
+}

gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/HtmlUnitTimeoutMillis.java → gwtp-core/gwtp-crawler-guice/src/main/java/com/gwtplatform/crawler/server/guice/service/HtmlUnitTimeoutMillis.java

@@ -14,7 +14,7 @@
  * the License.
  */
 
-package com.gwtplatform.crawlerservice.server;
+package com.gwtplatform.crawler.server.guice.service;
 
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;

gwtp-core/gwtp-crawler-spring/pom.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.gwtplatform</groupId>
+        <artifactId>gwtp-core</artifactId>
+        <version>1.6-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>gwtp-crawler-spring</artifactId>
+    <name>GWTP Crawler for Spring</name>
+
+    <build>
+        <resources>
+            <!-- Bundle sources with the jar, so they are visible to GWT's compiler -->
+            <resource>
+                <directory>src/main/java</directory>
+                <includes>
+                    <include>**/*.java</include>
+                </includes>
+            </resource>
+            <!-- Bundle module descriptor with the jar, so it is visible to GWT's compiler -->
+            <resource>
+                <directory>src/main/resources</directory>
+                <includes>
+                    <include>**/*.gwt.xml</include>
+                </includes>
+            </resource>
+        </resources>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>gwtp-crawler</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>javax.servlet</groupId>
+            <artifactId>servlet-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-core</artifactId>
+            <version>${spring.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context</artifactId>
+            <version>${spring.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-beans</artifactId>
+            <version>${spring.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-web</artifactId>
+            <version>${spring.version}</version>
+        </dependency>
+
+        <!-- Test dependencies -->
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-test</artifactId>
+            <version>${spring.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+</project>

gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/AbstractCrawlerModule.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server.spring;
+
+import java.util.logging.Logger;
+
+import org.springframework.context.annotation.Bean;
+
+/**
+ * Abstract crawler module for {@link @Configuration} setup.
+ * @author Ben Dol
+ */
+public abstract class AbstractCrawlerModule {
+    @Bean
+    protected Logger crawlLogger() {
+        return Logger.getAnonymousLogger();
+    }
+
+    @Bean
+    protected abstract String crawlKey();
+}

gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/filter/AbstractCrawlFilterModule.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server.spring.filter;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+
+import com.gwtplatform.crawler.server.spring.AbstractCrawlerModule;
+
+/**
+ * Abstract crawl filter module for {@link @Configuration} setup.
+ * @author Ben Dol
+ */
+@ComponentScan(basePackages = {
+        "com.gwtplatform.crawler.server.spring.filter"
+        })
+public abstract class AbstractCrawlFilterModule extends AbstractCrawlerModule {
+
+    @Bean
+    protected abstract String serviceUrl();
+}