Changeset dbd2c3

Timestamp:

02/04/08 16:11:49 (4 years ago)

Author:

Tomash Brechko <tomash.brechko@…>

Branches:

master-v0.7, master-v0.6, memcached_gzip, upstream_count_limit

Children:

b649a6, f54237

Parents:

2a28c1 (diff), 95d817 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

git-author:

Tomash Brechko <tomash.brechko@…> (02/04/08 16:11:49)

git-committer:

Tomash Brechko <tomash.brechko@…> (02/04/08 16:11:49)

Message:

Merge branch 'gunzip' into memcached_gzip

Location:

server/src/http/modules

Files:

• ngx_http_gzip_filter_module.c (modified) (8 diffs)
• ngx_http_memcached_module.c (modified) (7 diffs)

server/src/http/modules/ngx_http_gzip_filter_module.c

@@ -309 +309 @@
     ngx_http_gzip_ctx_t   *ctx;
     ngx_http_gzip_conf_t  *conf;
+    const char            *method = (r->gunzip ? "inflate" : "deflate");
 
     ctx = ngx_http_get_module_ctx(r, ngx_http_gzip_filter_module);
@@ -366 +367 @@
         if (rc != Z_OK) {
             ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
-                          "deflateInit2() failed: %d", rc);
+                          "%sInit2() failed: %d", method, rc);
             ngx_http_gzip_error(ctx);
             return NGX_ERROR;
@@ -456 +457 @@
 
                 if (ctx->in_buf->last_buf) {
-                    ctx->flush = Z_FINISH;
+                    /*
+                     * We use Z_BLOCK below for the following reason:
+                     * if we would use Z_FINISH, then decompression
+                     * may return Z_BUF_ERROR, meaning there wasn't
+                     * enough room for decompressed data.  This error
+                     * is not fatal according to zlib.h, however
+                     * ignoring it is dangerous and may mask real
+                     * bugs.  For instance, sometimes completely empty
+                     * last buffer is passed to this filter, and
+                     * decompression would enter the infinite loop: no
+                     * progress is possible because input is void and
+                     * Z_BUF_ERROR is ignored.  We can't use
+                     * Z_NO_FLUSH, because it will never return
+                     * Z_STREAM_END.  We can't use Z_SYNC_FLUSH,
+                     * because it has a special meaning.  So we use
+                     * Z_BLOCK, which eventually would return
+                     * Z_STREAM_END.
+                     *
+                     * Perhaps the above is also true for compression,
+                     * but right now we won't try to change the old
+                     * behaviour.
+                     */
+                    ctx->flush = (!r->gunzip ? Z_FINISH : Z_BLOCK);
 
                 } else if (ctx->in_buf->flush) {
@@ -474 +497 @@
 
 
-            /* is there a space for the gzipped data ? */
+            /* is there a space for the output data ? */
 
             if (ctx->zstream.avail_out == 0) {
@@ -503 +526 @@
             }
 
-            ngx_log_debug6(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                         "deflate in: ni:%p no:%p ai:%ud ao:%ud fl:%d redo:%d",
+            ngx_log_debug7(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                         "%s in: ni:%p no:%p ai:%ud ao:%ud fl:%d redo:%d",
+                         method,
                          ctx->zstream.next_in, ctx->zstream.next_out,
                          ctx->zstream.avail_in, ctx->zstream.avail_out,
@@ -515 +539 @@
             }
 
-            if (rc != Z_OK && rc != Z_STREAM_END && rc != Z_BUF_ERROR) {
+            if (rc != Z_OK && rc != Z_STREAM_END) {
                 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
-                              "deflate() failed: %d, %d", ctx->flush, rc);
+                              "%s() failed: %d, %d", method, ctx->flush, rc);
                 ngx_http_gzip_error(ctx);
                 return NGX_ERROR;
             }
 
-            ngx_log_debug5(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                           "deflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d",
+            ngx_log_debug6(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                           "%s out: ni:%p no:%p ai:%ud ao:%ud rc:%d",
+                           method,
                            ctx->zstream.next_in, ctx->zstream.next_out,
                            ctx->zstream.avail_in, ctx->zstream.avail_out,
                            rc);
 
-            ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                           "gzip in_buf:%p pos:%p",
-                           ctx->in_buf, ctx->in_buf->pos);
+            ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                           "%s in_buf:%p pos:%p",
+                           method, ctx->in_buf, ctx->in_buf->pos);
 
 
@@ -588 +613 @@
                 if (rc != Z_OK) {
                     ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
-                                  "deflateEnd() failed: %d", rc);
+                                  "%sEnd() failed: %d", method, rc);
                     ngx_http_gzip_error(ctx);
                     return NGX_ERROR;
@@ -601 +626 @@
                 }
 
-                cl->buf = ctx->out_buf;
-                cl->next = NULL;
-                *ctx->last_out = cl;
-                ctx->last_out = &cl->next;
+                /*
+                 * On decompression we could already output everything
+                 * under (ctx->flush == Z_SYNC_FLUSH), so we test here
+                 * that the buffer is not empty.
+                 */
+                if (!r->gunzip || ctx->out_buf->last > ctx->out_buf->pos) {
+                    cl->buf = ctx->out_buf;
+                    cl->next = NULL;
+                    *ctx->last_out = cl;
+                    ctx->last_out = &cl->next;
+                }
 
                 if (!r->gunzip) {

server/src/http/modules/ngx_http_memcached_module.c

@@ -14 +14 @@
     ngx_http_upstream_conf_t   upstream;
     ngx_int_t                  index;
+    ngx_uint_t                 gzip_flag;
 } ngx_http_memcached_loc_conf_t;
 
@@ -99 +100 @@
       offsetof(ngx_http_memcached_loc_conf_t, upstream.next_upstream),
       &ngx_http_memcached_next_upstream_masks },
+
+    { ngx_string("memcached_gzip_flag"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_memcached_loc_conf_t, gzip_flag),
+      NULL },
 
     { ngx_string("memcached_upstream_max_fails"),
@@ -298 +306 @@
 ngx_http_memcached_process_header(ngx_http_request_t *r)
 {
-    u_char                    *p, *len;
+    u_char                    *p, *beg;
     ngx_str_t                  line;
     ngx_http_upstream_t       *u;
     ngx_http_memcached_ctx_t  *ctx;
+    ngx_http_memcached_loc_conf_t  *mlcf;
+    uint32_t                   flags;
+    ngx_table_elt_t           *h;
+    ngx_http_core_loc_conf_t  *clcf;
 
     u = r->upstream;
@@ -346 +358 @@
         }
 
-        /* skip flags */
-
-        while (*p) {
-            if (*p++ == ' ') {
-                goto length;
-            }
+        beg = p;
+
+        while (*p && *p++ != ' ') { /* void */ }
+
+        if (! *p) {
+            goto no_valid;
         }
 
-        goto no_valid;
-
-    length:
-
-        len = p;
+        flags = ngx_atoof(beg, p - beg - 1);
+
+        beg = p;
 
         while (*p && *p++ != CR) { /* void */ }
 
-        r->headers_out.content_length_n = ngx_atoof(len, p - len - 1);
+        r->headers_out.content_length_n = ngx_atoof(beg, p - beg - 1);
         if (r->headers_out.content_length_n == -1) {
             ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
@@ -369 +379 @@
                           &line, &ctx->key);
             return NGX_HTTP_UPSTREAM_INVALID_HEADER;
+        }
+
+        mlcf = ngx_http_get_module_loc_conf(r, ngx_http_memcached_module);
+
+        if (flags & mlcf->gzip_flag) {
+#if (NGX_HTTP_GZIP)
+            clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+
+            if (ngx_http_gzip_ok(r) == NGX_OK) {
+                h = ngx_list_push(&r->headers_out.headers);
+                if (h == NULL) {
+                    return NGX_ERROR;
+                }
+
+                h->hash = 1;
+                h->key.len = sizeof("Content-Encoding") - 1;
+                h->key.data = (u_char *) "Content-Encoding";
+                h->value.len = sizeof("gzip") - 1;
+                h->value.data = (u_char *) "gzip";
+
+                r->headers_out.content_encoding = h;
+
+                if (clcf->gzip_vary) {
+                    h = ngx_list_push(&r->headers_out.headers);
+                    if (h == NULL) {
+                        return NGX_ERROR;
+                    }
+
+                    h->hash = 1;
+                    h->key.len = sizeof("Vary") - 1;
+                    h->key.data = (u_char *) "Vary";
+                    h->value.len = sizeof("Accept-Encoding") - 1;
+                    h->value.data = (u_char *) "Accept-Encoding";
+                }
+            } else {
+                if (clcf->gunzip) {
+                    r->gunzip = 1;
+                } else {
+#endif
+                    /*
+                     * If the client can't accept compressed data, and
+                     * automatic decompression is not enabled, we
+                     * return 404 in the hope that the next upstream
+                     * will return uncompressed data.
+                     */
+                    u->headers_in.status_n = 404;
+                    u->state->status = 404;
+
+                    return NGX_OK;
+#if (NGX_HTTP_GZIP)
+                }
+            }
+#endif
         }
 
@@ -532 +595 @@
 
     conf->upstream.buffer_size = NGX_CONF_UNSET_SIZE;
+
+    conf->gzip_flag = NGX_CONF_UNSET_UINT;
 
     /* the hardcoded values */
@@ -576 +641 @@
                                |NGX_HTTP_UPSTREAM_FT_TIMEOUT));
 
+    ngx_conf_merge_uint_value(conf->gzip_flag,
+                              prev->gzip_flag, 0);
+
     if (conf->upstream.next_upstream & NGX_HTTP_UPSTREAM_FT_OFF) {
         conf->upstream.next_upstream = NGX_CONF_BITMASK_SET