Merge with main repo.

9 files changed, 210 insertions, 86 deletions

diff --git a/src/IO/Url.h b/src/IO/Url.h
index 15934e13..d9333d67 100644
--- a/src/IO/Url.h
+++ b/src/IO/Url.h
@@ -19,7 +19,7 @@ void a_Http_set_proxy_passwd(const char *str);
 char *a_Http_make_connect_str(const DilloUrl *url);
 const char *a_Http_get_proxy_urlstr();
 Dstr *a_Http_make_query_str(const DilloUrl *url, const DilloUrl *requester,
-                            bool_t use_proxy);
+                            int web_flags, bool_t use_proxy);
 
 void a_Http_ccc (int Op, int Branch, int Dir, ChainLink *Info,
                  void *Data1, void *Data2);
diff --git a/src/IO/http.c b/src/IO/http.c
index 1af1996d..a0021a9e 100644
--- a/src/IO/http.c
+++ b/src/IO/http.c
@@ -262,13 +262,19 @@ static Dstr *Http_make_content_type(const DilloUrl *url)
  * Make the http query string
  */
 Dstr *a_Http_make_query_str(const DilloUrl *url, const DilloUrl *requester,
-                            bool_t use_proxy)
+                            int web_flags, bool_t use_proxy)
 {
    char *ptr, *cookies, *referer, *auth;
    Dstr *query      = dStr_new(""),
         *request_uri = dStr_new(""),
         *proxy_auth = dStr_new("");
 
+   /* BUG: dillo doesn't actually understand application/xml yet */
+   const char *accept_hdr_value =
+      web_flags & WEB_Image ? "image/png,image/*;q=0.8,*/*;q=0.5" :
+      web_flags & WEB_Stylesheet ? "text/css,*/*;q=0.1" :
+      "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
+
    if (use_proxy) {
       dStr_sprintfa(request_uri, "%s%s",
                     URL_STR(url),
@@ -294,24 +300,23 @@ Dstr *a_Http_make_query_str(const DilloUrl *url, const DilloUrl *requester,
       dStr_sprintfa(
          query,
          "POST %s HTTP/1.1\r\n"
-         "Connection: close\r\n"
-         "Accept: text/*,image/*,*/*;q=0.2\r\n"
-         "Accept-Charset: utf-8,*;q=0.8\r\n"
-         "Accept-Encoding: gzip\r\n"
+         "Host: %s\r\n"
+         "User-Agent: %s\r\n"
+         "Accept: %s\r\n"
          "%s" /* language */
+         "Accept-Encoding: gzip, deflate\r\n"
          "%s" /* auth */
          "DNT: 1\r\n"
-         "Host: %s\r\n"
-         "%s"
-         "%s"
-         "User-Agent: %s\r\n"
-         "Content-Length: %ld\r\n"
+         "%s" /* proxy auth */
+         "%s" /* referer */
+         "Connection: close\r\n"
          "Content-Type: %s\r\n"
+         "Content-Length: %ld\r\n"
          "%s" /* cookies */
          "\r\n",
-         request_uri->str, HTTP_Language_hdr, auth ? auth : "",
-         URL_AUTHORITY(url), proxy_auth->str, referer, prefs.http_user_agent,
-         (long)URL_DATA(url)->len, content_type->str,
+         request_uri->str, URL_AUTHORITY(url), prefs.http_user_agent,
+         accept_hdr_value, HTTP_Language_hdr, auth ? auth : "",
+         proxy_auth->str, referer, content_type->str, (long)URL_DATA(url)->len,
          cookies);
       dStr_append_l(query, URL_DATA(url)->str, URL_DATA(url)->len);
       dStr_free(content_type, TRUE);
@@ -319,25 +324,25 @@ Dstr *a_Http_make_query_str(const DilloUrl *url, const DilloUrl *requester,
       dStr_sprintfa(
          query,
          "GET %s HTTP/1.1\r\n"
-         "%s"
-         "Connection: close\r\n"
-         "Accept: text/*,image/*,*/*;q=0.2\r\n"
-         "Accept-Charset: utf-8,*;q=0.8\r\n"
-         "Accept-Encoding: gzip\r\n"
+         "Host: %s\r\n"
+         "User-Agent: %s\r\n"
+         "Accept: %s\r\n"
          "%s" /* language */
+         "Accept-Encoding: gzip, deflate\r\n"
          "%s" /* auth */
          "DNT: 1\r\n"
-         "Host: %s\r\n"
-         "%s"
-         "%s"
-         "User-Agent: %s\r\n"
+         "%s" /* proxy auth */
+         "%s" /* referer */
+         "Connection: close\r\n"
+         "%s" /* cache control */
          "%s" /* cookies */
          "\r\n",
-         request_uri->str,
+         request_uri->str, URL_AUTHORITY(url), prefs.http_user_agent,
+         accept_hdr_value, HTTP_Language_hdr, auth ? auth : "",
+         proxy_auth->str, referer,
          (URL_FLAGS(url) & URL_E2EQuery) ?
-            "Cache-Control: no-cache\r\nPragma: no-cache\r\n" : "",
-         HTTP_Language_hdr, auth ? auth : "", URL_AUTHORITY(url),
-         proxy_auth->str, referer, prefs.http_user_agent, cookies);
+            "Pragma: no-cache\r\nCache-Control: no-cache\r\n" : "",
+         cookies);
    }
    dFree(referer);
    dFree(cookies);
@@ -358,7 +363,7 @@ static void Http_send_query(ChainLink *Info, SocketData_t *S)
    DataBuf *dbuf;
 
    /* Create the query */
-   query = a_Http_make_query_str(S->web->url, S->web->requester,
+   query = a_Http_make_query_str(S->web->url, S->web->requester, S->web->flags,
                                  S->flags & HTTP_SOCKET_USE_PROXY);
    dbuf = a_Chain_dbuf_new(query->str, query->len, 0);
 
diff --git a/src/capi.c b/src/capi.c
index de338267..531ae34d 100644
--- a/src/capi.c
+++ b/src/capi.c
@@ -301,7 +301,8 @@ static char *Capi_dpi_build_cmd(DilloWeb *web, char *server)
    if (strcmp(server, "proto.https") == 0) {
       /* Let's be kind and make the HTTP query string for the dpi */
       char *proxy_connect = a_Http_make_connect_str(web->url);
-      Dstr *http_query = a_Http_make_query_str(web->url, web->requester,FALSE);
+      Dstr *http_query = a_Http_make_query_str(web->url, web->requester,
+                                               web->flags, FALSE);
 
       if ((uint_t) http_query->len > strlen(http_query->str)) {
          /* Can't handle NULLs embedded in query data */
diff --git a/src/css.cc b/src/css.cc
index 4938a1cf..5bdf4fdb 100644
--- a/src/css.cc
+++ b/src/css.cc
@@ -399,9 +399,9 @@ void CssStyleSheet::addRule (CssRule *rule) {
    }
 
    if (ruleList) {
-      rule->selector->setMatchCacheOffset (matchCacheOffset);
-      matchCacheOffset += rule->selector->size ();
       ruleList->insert (rule);
+      if (rule->selector->getRequiredMatchCache () > requiredMatchCache)
+         requiredMatchCache = rule->selector->getRequiredMatchCache ();
    } else {
       assert (top->getElement () == CssSimpleSelector::ELEMENT_NONE);
       delete rule;
@@ -488,7 +488,7 @@ CssStyleSheet CssContext::userAgentSheet;
 
 CssContext::CssContext () {
    pos = 0;
-   matchCache[CSS_PRIMARY_USER_AGENT].setSize (userAgentSheet.matchCacheOffset, -1);
+   matchCache.setSize (userAgentSheet.getRequiredMatchCache (), -1);
 }
 
 /**
@@ -505,28 +505,25 @@ void CssContext::apply (CssPropertyList *props, Doctree *docTree,
          CssPropertyList *tagStyle, CssPropertyList *tagStyleImportant,
          CssPropertyList *nonCssHints) {
 
-   userAgentSheet.apply (props, docTree, node,
-                         &matchCache[CSS_PRIMARY_USER_AGENT]);
-   sheet[CSS_PRIMARY_USER].apply (props, docTree, node,
-                                  &matchCache[CSS_PRIMARY_USER]);
+   userAgentSheet.apply (props, docTree, node, &matchCache);
+
+   sheet[CSS_PRIMARY_USER].apply (props, docTree, node, &matchCache);
 
    if (nonCssHints)
         nonCssHints->apply (props);
 
-   sheet[CSS_PRIMARY_AUTHOR].apply (props, docTree, node,
-                                    &matchCache[CSS_PRIMARY_AUTHOR]);
+   sheet[CSS_PRIMARY_AUTHOR].apply (props, docTree, node, &matchCache);
 
    if (tagStyle)
         tagStyle->apply (props);
 
    sheet[CSS_PRIMARY_AUTHOR_IMPORTANT].apply (props, docTree, node,
-                                     &matchCache[CSS_PRIMARY_AUTHOR_IMPORTANT]);
+                                              &matchCache);
 
    if (tagStyleImportant)
         tagStyleImportant->apply (props);
 
-   sheet[CSS_PRIMARY_USER_IMPORTANT].apply (props, docTree, node,
-                                     &matchCache[CSS_PRIMARY_USER_IMPORTANT]);
+   sheet[CSS_PRIMARY_USER_IMPORTANT].apply (props, docTree, node, &matchCache);
 }
 
 void CssContext::addRule (CssSelector *sel, CssPropertyList *props,
@@ -540,12 +537,16 @@ void CssContext::addRule (CssSelector *sel, CssPropertyList *props,
            !rule->isSafe ()) {
          MSG_WARN ("Ignoring unsafe author style that might reveal browsing history\n");
          delete rule;
-      } else if (order == CSS_PRIMARY_USER_AGENT) {
-         userAgentSheet.addRule (rule);
-         matchCache[CSS_PRIMARY_USER_AGENT].setSize (userAgentSheet.matchCacheOffset, -1);
-      } else { 
-         sheet[order].addRule (rule);
-         matchCache[order].setSize (sheet[order].matchCacheOffset, -1);
+      } else {
+         rule->selector->setMatchCacheOffset(matchCache.size ());
+         if (rule->selector->getRequiredMatchCache () > matchCache.size ())
+            matchCache.setSize (rule->selector->getRequiredMatchCache (), -1);
+
+         if (order == CSS_PRIMARY_USER_AGENT) {
+            userAgentSheet.addRule (rule);
+         } else { 
+            sheet[order].addRule (rule);
+         }
       }
    }
 }
diff --git a/src/css.hh b/src/css.hh
index 4dd94ffe..22e7e700 100644
--- a/src/css.hh
+++ b/src/css.hh
@@ -143,6 +143,7 @@ inline float CSS_LENGTH_VALUE (CssLength l) {
 }
 
 typedef enum {
+   CSS_PROPERTY_END = -1, // used as terminator in CssShorthandInfo
    CSS_PROPERTY_BACKGROUND_ATTACHMENT,
    CSS_PROPERTY_BACKGROUND_COLOR,
    CSS_PROPERTY_BACKGROUND_IMAGE,
@@ -401,16 +402,20 @@ class CssSelector {
       void addSimpleSelector (Combinator c);
       inline CssSimpleSelector *top () {
          return selectorList.getRef (selectorList.size () - 1)->selector;
-      };
+      }
       inline int size () { return selectorList.size (); };
       inline bool match (Doctree *dt, const DoctreeNode *node,
                          MatchCache *matchCache) {
          return match (dt, node, selectorList.size () - 1, COMB_NONE,
                        matchCache);
-      };
+      }
       inline void setMatchCacheOffset (int mo) {
-         matchCacheOffset = mo;
-      };
+         if (matchCacheOffset == -1)
+            matchCacheOffset = mo;
+      }
+      inline int getRequiredMatchCache () {
+         return matchCacheOffset + size ();
+      }
       int specificity ();
       bool checksPseudoClass ();
       void print ();
@@ -483,14 +488,14 @@ class CssStyleSheet {
 
       RuleList elementTable[ntags], anyTable;
       RuleMap idTable, classTable;
+      int requiredMatchCache;
 
    public:
-      int matchCacheOffset;
-
-      CssStyleSheet () { matchCacheOffset = 0; }
+      CssStyleSheet () { requiredMatchCache = 0; }
       void addRule (CssRule *rule);
       void apply (CssPropertyList *props, Doctree *docTree,
                   const DoctreeNode *node, MatchCache *matchCache) const;
+      int getRequiredMatchCache () { return requiredMatchCache; }
 };
 
 /**
@@ -500,7 +505,7 @@ class CssContext {
    private:
       static CssStyleSheet userAgentSheet;
       CssStyleSheet sheet[CSS_PRIMARY_USER_IMPORTANT + 1];
-      MatchCache matchCache[CSS_PRIMARY_USER_IMPORTANT + 1];
+      MatchCache matchCache;
       int pos;
 
    public:
diff --git a/src/cssparser.cc b/src/cssparser.cc
index 07412a8d..04faffa2 100644
--- a/src/cssparser.cc
+++ b/src/cssparser.cc
@@ -288,7 +288,8 @@ typedef struct {
       CSS_SHORTHAND_FONT,       /* special, used for 'font' */
    } type;
    const CssPropertyName *properties; /* CSS_SHORTHAND_MULTIPLE:
-                                       *   must be terminated by -1
+                                       *   must be terminated by
+                                       *   CSS_PROPERTY_END 
                                        * CSS_SHORTHAND_DIRECTIONS:
                                        *   must have length 4
                                        * CSS_SHORTHAND_BORDERS:
@@ -303,14 +304,14 @@ const CssPropertyName Css_background_properties[] = {
    CSS_PROPERTY_BACKGROUND_REPEAT,
    CSS_PROPERTY_BACKGROUND_ATTACHMENT,
    CSS_PROPERTY_BACKGROUND_POSITION,
-   (CssPropertyName) - 1
+   CSS_PROPERTY_END
 };
 
 const CssPropertyName Css_border_bottom_properties[] = {
    CSS_PROPERTY_BORDER_BOTTOM_WIDTH,
    CSS_PROPERTY_BORDER_BOTTOM_STYLE,
    CSS_PROPERTY_BORDER_BOTTOM_COLOR,
-   (CssPropertyName) - 1
+   CSS_PROPERTY_END
 };
 
 const CssPropertyName Css_border_color_properties[4] = {
@@ -324,14 +325,14 @@ const CssPropertyName Css_border_left_properties[] = {
    CSS_PROPERTY_BORDER_LEFT_WIDTH,
    CSS_PROPERTY_BORDER_LEFT_STYLE,
    CSS_PROPERTY_BORDER_LEFT_COLOR,
-   (CssPropertyName) - 1
+   CSS_PROPERTY_END
 };
 
 const CssPropertyName Css_border_right_properties[] = {
    CSS_PROPERTY_BORDER_RIGHT_WIDTH,
    CSS_PROPERTY_BORDER_RIGHT_STYLE,
    CSS_PROPERTY_BORDER_RIGHT_COLOR,
-   (CssPropertyName) - 1
+   CSS_PROPERTY_END
 };
 
 const CssPropertyName Css_border_style_properties[] = {
@@ -345,7 +346,7 @@ const CssPropertyName Css_border_top_properties[] = {
    CSS_PROPERTY_BORDER_TOP_WIDTH,
    CSS_PROPERTY_BORDER_TOP_STYLE,
    CSS_PROPERTY_BORDER_TOP_COLOR,
-   (CssPropertyName) - 1
+   CSS_PROPERTY_END
 };
 
 const CssPropertyName Css_border_width_properties[] = {
@@ -359,7 +360,7 @@ const CssPropertyName Css_list_style_properties[] = {
    CSS_PROPERTY_LIST_STYLE_TYPE,
    CSS_PROPERTY_LIST_STYLE_POSITION,
    CSS_PROPERTY_LIST_STYLE_IMAGE,
-   (CssPropertyName) - 1
+   CSS_PROPERTY_END
 };
 
 const CssPropertyName Css_margin_properties[] = {
@@ -373,7 +374,7 @@ const CssPropertyName Css_outline_properties[] = {
    CSS_PROPERTY_OUTLINE_COLOR,
    CSS_PROPERTY_OUTLINE_STYLE,
    CSS_PROPERTY_OUTLINE_WIDTH,
-   (CssPropertyName) - 1
+   CSS_PROPERTY_END
 };
 
 const CssPropertyName Css_padding_properties[] = {
@@ -404,7 +405,7 @@ const CssPropertyName Css_font_properties[] = {
    CSS_PROPERTY_FONT_VARIANT,
    CSS_PROPERTY_FONT_WEIGHT,
    CSS_PROPERTY_FONT_FAMILY,
-   (CssPropertyName) - 1
+   CSS_PROPERTY_END
 };
 
 static const CssShorthandInfo Css_shorthand_info[] = {
@@ -1260,7 +1261,8 @@ void CssParser::parseDeclaration(CssPropertyList *props,
                   do {
                      for (found = false, i = 0;
                           !found &&
-                          Css_shorthand_info[sh_index].properties[i] != -1;
+                          Css_shorthand_info[sh_index].properties[i] !=
+                          CSS_PROPERTY_END;
                           i++)
                         if (tokenMatchesProperty(Css_shorthand_info[sh_index].
                                                  properties[i], &type)) {
diff --git a/src/decode.c b/src/decode.c
index c5752e62..69c6423a 100644
--- a/src/decode.c
+++ b/src/decode.c
@@ -86,6 +86,20 @@ static void Decode_chunked_free(Decode *dc)
    dStr_free(dc->leftover, 1);
 }
 
+static void Decode_compression_free(Decode *dc)
+{
+   (void)inflateEnd((z_stream *)dc->state);
+
+   dFree(dc->state);
+   dFree(dc->buffer);
+}
+
+/*
+ * BUG: A fair amount of duplicated code exists in the gzip/deflate decoding,
+ * but an attempt to pull out the common code left everything too contorted
+ * for what it accomplished.
+ */
+
 /*
  * Decode gzipped data
  */
@@ -122,12 +136,92 @@ static Dstr *Decode_gzip(Decode *dc, const char *instr, int inlen)
    return output;
 }
 
-static void Decode_gzip_free(Decode *dc)
+/*
+ * Decode (raw) deflated data
+ */
+static Dstr *Decode_raw_deflate(Decode *dc, const char *instr, int inlen)
 {
-   (void)inflateEnd((z_stream *)dc->state);
+   int rc = Z_OK;
 
-   dFree(dc->state);
-   dFree(dc->buffer);
+   z_stream *zs = (z_stream *)dc->state;
+
+   int inputConsumed = 0;
+   Dstr *output = dStr_new("");
+
+   while ((rc == Z_OK) && (inputConsumed < inlen)) {
+      zs->next_in = (Bytef *)instr + inputConsumed;
+      zs->avail_in = inlen - inputConsumed;
+
+      zs->next_out = (Bytef *)dc->buffer;
+      zs->avail_out = bufsize;
+
+      rc = inflate(zs, Z_SYNC_FLUSH);
+
+      dStr_append_l(output, dc->buffer, zs->total_out);
+
+      if ((rc == Z_OK) || (rc == Z_STREAM_END)) {
+         // Z_STREAM_END at end of file
+
+         inputConsumed += zs->total_in;
+         zs->total_out = 0;
+         zs->total_in = 0;
+      } else if (rc == Z_DATA_ERROR) {
+         MSG_ERR("raw deflate decompression also failed\n");
+      }
+   }
+   return output;
+}
+
+/*
+ * Decode deflated data, initially presuming that the required zlib wrapper
+ * is there. On data error, switch to Decode_raw_deflate().
+ */
+static Dstr *Decode_deflate(Decode *dc, const char *instr, int inlen)
+{
+   int rc = Z_OK;
+
+   z_stream *zs = (z_stream *)dc->state;
+
+   int inputConsumed = 0;
+   Dstr *output = dStr_new("");
+
+   while ((rc == Z_OK) && (inputConsumed < inlen)) {
+      zs->next_in = (Bytef *)instr + inputConsumed;
+      zs->avail_in = inlen - inputConsumed;
+
+      zs->next_out = (Bytef *)dc->buffer;
+      zs->avail_out = bufsize;
+
+      rc = inflate(zs, Z_SYNC_FLUSH);
+
+      dStr_append_l(output, dc->buffer, zs->total_out);
+
+      if ((rc == Z_OK) || (rc == Z_STREAM_END)) {
+         // Z_STREAM_END at end of file
+
+         inputConsumed += zs->total_in;
+         zs->total_out = 0;
+         zs->total_in = 0;
+      } else if (rc == Z_DATA_ERROR) {
+         MSG_WARN("Deflate decompression error. Certain servers illegally fail"
+                 " to send data in a zlib wrapper. Let's try raw deflate.\n");
+         dStr_free(output, 1);
+         (void)inflateEnd(zs);
+         dFree(dc->state);
+         dc->state = zs = dNew(z_stream, 1);;
+         zs->zalloc = NULL;
+         zs->zfree = NULL;
+         zs->next_in = NULL;
+         zs->avail_in = 0;
+         dc->decode = Decode_raw_deflate;
+
+         // Negative value means that we want raw deflate.
+         inflateInit2(zs, -MAX_WBITS);
+
+         return Decode_raw_deflate(dc, instr, inlen);
+      }
+   }
+   return output;
 }
 
 /*
@@ -204,36 +298,50 @@ Decode *a_Decode_transfer_init(const char *format)
    return dc;
 }
 
+static Decode *Decode_content_init_common()
+{
+   z_stream *zs = dNew(z_stream, 1);
+   Decode *dc = dNew(Decode, 1);
+
+   zs->zalloc = NULL;
+   zs->zfree = NULL;
+   zs->next_in = NULL;
+   zs->avail_in = 0;
+   dc->state = zs;
+   dc->buffer = dNew(char, bufsize);
+
+   dc->free = Decode_compression_free;
+   dc->leftover = NULL; /* not used */
+   return dc;
+}
+
 /*
- * Initialize content decoder. Currently handles gzip.
- *
- * zlib is also capable of handling "deflate"/zlib-encoded data, but web
- * servers have not standardized on whether to send such data with a header.
+ * Initialize content decoder. Currently handles 'gzip' and 'deflate'.
  */
 Decode *a_Decode_content_init(const char *format)
 {
+   z_stream *zs;
    Decode *dc = NULL;
 
    if (format && *format) {
       if (!dStrAsciiCasecmp(format, "gzip") ||
           !dStrAsciiCasecmp(format, "x-gzip")) {
-         z_stream *zs;
-         _MSG("gzipped data!\n");
-
-         dc = dNew(Decode, 1);
-         dc->buffer = dNew(char, bufsize);
-         dc->state = zs = dNew(z_stream, 1);
-         zs->zalloc = NULL;
-         zs->zfree = NULL;
-         zs->next_in = NULL;
-         zs->avail_in = 0;
+         MSG("gzipped data!\n");
 
+         dc = Decode_content_init_common();
+         zs = (z_stream *)dc->state;
          /* 16 is a magic number for gzip decoding */
          inflateInit2(zs, MAX_WBITS+16);
 
          dc->decode = Decode_gzip;
-         dc->free = Decode_gzip_free;
-         dc->leftover = NULL; /* not used */
+      } else if (!dStrAsciiCasecmp(format, "deflate")) {
+         MSG("deflated data!\n");
+
+         dc = Decode_content_init_common();
+         zs = (z_stream *)dc->state;
+         inflateInit(zs);
+
+         dc->decode = Decode_deflate;
       } else {
          MSG("Content-Encoding '%s' not recognized.\n", format);
       }
diff --git a/src/html.cc b/src/html.cc
index 3319b35e..f6233864 100644
--- a/src/html.cc
+++ b/src/html.cc
@@ -3258,6 +3258,7 @@ void a_Html_load_stylesheet(DilloHtml *html, DilloUrl *url)
       /* Fill a Web structure for the cache query */
       int ClientKey;
       DilloWeb *Web = a_Web_new(html->bw, url, html->page_url);
+      Web->flags |= WEB_Stylesheet;
       if ((ClientKey = a_Capi_open_url(Web, Html_css_load_callback, NULL))) {
          ++html->bw->NumPendingStyleSheets;
          a_Bw_add_client(html->bw, ClientKey, 0);
diff --git a/src/web.hh b/src/web.hh
index 2c22edb5..2e8fbf69 100644
--- a/src/web.hh
+++ b/src/web.hh
@@ -15,7 +15,8 @@ extern "C" {
  */
 #define WEB_RootUrl  1
 #define WEB_Image    2
-#define WEB_Download 4   /* Half implemented... */
+#define WEB_Stylesheet 4
+#define WEB_Download 8   /* Half implemented... */
 
 
 typedef struct _DilloWeb DilloWeb;