HttpResourceFetcher

+    package de.juplo.httpresources;
+
+    import org.slf4j.Logger;
+    import org.slf4j.LoggerFactory;
+    import org.springframework.cache.Cache;
+    import org.springframework.http.HttpHeaders;
+    import org.springframework.http.HttpMethod;
+    import org.springframework.http.HttpStatus;
+    import org.springframework.http.client.ClientHttpRequest;
+    import org.springframework.http.client.ClientHttpRequestFactory;
+    import org.springframework.http.client.ClientHttpResponse;
+    import org.springframework.util.FileCopyUtils;
+
+    import java.io.IOException;
+    import java.io.InputStream;
+    import java.net.URI;
+    import java.time.Clock;
+    import java.util.HashMap;
+    import java.util.Map;
+    import java.util.function.Predicate;
+
+    import static org.springframework.http.HttpHeaders.*;
+
+    public class HttpResourceFetcher
+    {
+      private final static Logger LOG =
+              LoggerFactory.getLogger(HttpResourceFetcher.class);
+
+      public final static int DEFAULT_TTL = 3600;
+
+      private final ClientHttpRequestFactory requestFactory;
+      private final Cache cache;
+      private final Clock clock;
+
+      private final Predicate<String> filter;
+
+      private final int defaultTTL;
+      private final int minTTL;
+
+      private final boolean serveStale;
+
+
+      public HttpResourceFetcher(ClientHttpRequestFactory factory, Cache cache, Clock clock)
+      {
+        this(factory, cache, clock, DEFAULT_TTL, 0, true);
+      }
+
+      /**
+       *
+       * @param factory
+       * @param clock
+       * @param defaultTTL default time to live in milliseconds
+       * Default TTL, if no according information is present in the HTTP-headers.
+       * If set to <code>0</code>, caching will be disabled, if no according
+       * HTTP-headers are present.
+       * @param minTTL the minimum time to live, if caching is enabled
+       * A minimum TTL, that will overwrite the time to live, that was extracted
+       * from the HTTP-headers.
+       * The minimum TTL is only applied, if the default TTL is set to a value
+       * greater than zero, hence enabling caching by default.
+       * @param serveStale
+       */
+      public HttpResourceFetcher(
+          ClientHttpRequestFactory factory,
+          Cache cache,
+          Clock clock,
+          int defaultTTL,
+          int minTTL,
+          boolean serveStale)
+      {
+        this.requestFactory = factory;
+        this.cache = cache;
+        this.clock = clock;
+        this.defaultTTL = defaultTTL;
+        this.minTTL = minTTL;
+        this.serveStale = serveStale;
+
+        /** Filter may pass on more headers and may also be configurable later */
+        filter = (key) -> CONTENT_TYPE.equalsIgnoreCase(key);
+      }
+
+
+      /**
+       * Fetches the remote resource and reports, if it was modified.
+       *
+       * This method fetches the remote resource, if was not already fetched.
+       * If the resource was already fetched, it revalidates it, if necessary.
+       *
+       * @return {@code true}, if the resource has changed or was fetched for
+       * the first time, otherwise {@code false}
+       */
+      public HttpData./de/juplo/httpresources/HttpData.html#HttpData">HttpData fetch(URI uri, HttpData data)
+      {
+        HttpData cached = cache.get(uri, HttpData.class);
+        if (cached != null && cached != data)
+        {
+          if (cached.expires >= clock.millis())
+            return cached;
+          else
+            data = cached;
+        }
+
+        boolean has_existed = data.content != null;
+        boolean must_revalidate = data.revalidate;
+
+        ClientHttpRequest request = createGetRequest(uri);
+        if (has_existed)
+        {
+          if (data.eTag != null)
+            request.getHeaders().setIfNoneMatch(data.eTag);
+          if (data.lastModified > 0)
+            request.getHeaders().setIfModifiedSince(data.lastModified);
+        }
+
+        HttpStatus status = HttpStatus.INTERNAL_SERVER_ERROR;
+        long request_time = clock.millis();
+
+        try (ClientHttpResponse response = request.execute())
+        {
+          status = response.getStatusCode();
+
+          LOG.debug("{} -- {}", status, uri);
+
+          switch (status)
+          {
+            case OK:
+
+              InputStream body = response.getBody();
+              byte[] content = FileCopyUtils.copyToByteArray(body);
+
+              data = readCachingHeaders(content, response, response.getHeaders(), request_time, defaultTTL);
+              cache.put(uri, data);
+              return data;
+
+            case NOT_MODIFIED:
+
+              data = readCachingHeaders(data.content, response, data.headers, request_time, defaultTTL);
+              cache.put(uri, data);
+              return data;
+
+
+            case NOT_FOUND:
+
+              // If stale data is served, 404-responses for already cached
+              // resources are quietly ignored
+              if (!serveStale)
+              {
+                data = readCachingHeaders(null, response, response.getHeaders(), request_time, 0);
+              }
+
+              cache.put(uri, data);
+              return data;
+
+            default:
+
+              // TODO:
+              // A client has at least to distinguish the status-classes, denoted
+              // by the first digit of the code.
+              // See: https://tools.ietf.org/html/rfc7231#section-6
+          }
+        }
+        catch (IOException e)
+        {
+          LOG.error("Cannot retrieve {}: {}", uri, e.toString());
+        }
+
+
+        if ((data.content == null) || (data.revalidate && !serveStale))
+        {
+          data = HttpData.SERVER_ERROR;
+        }
+        else
+        {
+          // An existing resource can be considered still existent, if the
+          // request fails and the resource does not have to be revalidated.
+          // It will also be considered as still existent, if revalidation is
+          // enforced through the HTTP-protocol, but serving stale data is
+          // enabled.
+        }
+
+        cache.put(uri, data);
+        return data;
+      }
+
+
+      private HttpData readCachingHeaders(
+          byte[] content,
+          ClientHttpResponse response,
+          HttpHeaders headers,
+          long request_time,
+          long defaultTTL
+          )
+      {
+        HttpHeaders responseHeaders = response.getHeaders();
+        // Calculating Freshness Lifetime
+        // See: https://tools.ietf.org/html/rfc7234#section-4.2.1
+
+        long now = clock.millis();
+
+        long date_value = responseHeaders.getDate();
+        long apparent_age = 0L;
+
+        if (date_value < 0L)
+        {
+          // Use the current time, if no date-header is present
+          // See: https://tools.ietf.org/html/rfc7231#section-7.1.1.2
+          date_value = now;
+        }
+        else
+        {
+          apparent_age = now - date_value;
+          apparent_age = apparent_age < 0 ? 0 : apparent_age;
+        }
+
+        long corrected_age_value = 0L;
+        String value = responseHeaders.getFirst(AGE);
+        if (value != null)
+        {
+          try
+          {
+            long response_delay = now - request_time;
+            long age_value = Long.parseLong(value) * 1000; // convert s to ms
+            age_value = age_value < 0 ? 0 : age_value;
+            corrected_age_value = age_value + response_delay;
+          }
+          catch (NumberFormatException e) {}
+        }
+
+        long corrected_initial_age =
+            apparent_age > corrected_age_value
+                ? apparent_age
+                : corrected_age_value;
+
+        long lastModified = responseHeaders.getLastModified();
+        lastModified = lastModified < 0L ? 0 : lastModified;
+
+        String eTag = responseHeaders.getETag();
+
+
+        Map<String, String> directives = new HashMap<>();
+        for (String field : responseHeaders.getOrEmpty(CACHE_CONTROL))
+        {
+          for (String directive : field.split("\\s*,\\s*"))
+          {
+            String[] splitted = directive.split("\\s*=\\s*");
+            if (splitted.length > 2)
+            {
+              LOG.warn("Ingoring garbled directive: {}", directive);
+              continue;
+            }
+            directives.put(
+                splitted[0].trim(),
+                splitted.length == 2 ? splitted[1] : null
+                 );
+          }
+        }
+
+        long ttl = -1l;
+        try
+        {
+          String maxAge = directives.get("max-age");
+          if (maxAge != null)
+            ttl = Long.parseUnsignedLong(maxAge) *1000;
+        }
+        catch (NumberFormatException e) {}
+
+        boolean revalidate = directives.containsKey("must-revalidate");
+        long expires = 0l;
+
+        if (!directives.containsKey("no-cache") ||
+            directives.get("no-cache") != null
+            )
+        {
+          if (ttl < 0)
+          {
+            if (responseHeaders.get(EXPIRES) != null)
+            {
+              expires = responseHeaders.getExpires();
+              // If the "Expires"-header cannot be parsed to a valid date, it has
+              // to be interpreted as a time in the past.
+              // See: https://tools.ietf.org/html/rfc7234#section-5.3
+              expires = expires == -1 ? 0 : expires;
+              return new HttpData(content, filter(headers), revalidate, lastModified, expires, eTag);
+            }
+
+            ttl = defaultTTL;
+          }
+
+          expires = date_value + (minTTL > ttl && defaultTTL > 0 ? minTTL : ttl) - corrected_initial_age;
+          return new HttpData(content, filter(headers), revalidate, lastModified, expires, eTag);
+        }
+
+        expires = minTTL == 0 || defaultTTL == 0 ? date_value : date_value + minTTL - corrected_initial_age;
+        return new HttpData(content, filter(headers), revalidate, lastModified, expires, eTag);
+      }
+
+      private HttpHeaders filter(HttpHeaders headers)
+      {
+        HttpHeaders result = new HttpHeaders();
+        headers.forEach((key, value) ->
+        {
+          if (filter.test(key))
+            result.put(key, value);
+        });
+        return result;
+      }
+
+      private ClientHttpRequest createGetRequest(URI uri)
+      {
+        try
+        {
+          ClientHttpRequest request = requestFactory.createRequest(uri, HttpMethod.GET);
+          request.getHeaders().set(ACCEPT_ENCODING, "gzip");
+          return request;
+        }
+        catch (IOException e)
+        {
+          throw new RuntimeException(e);
+        }
+      }
+    }
+