Changeset 2a12f7

Timestamp:

02/04/08 16:12:02 (4 years ago)

Author:

Tomash Brechko <tomash.brechko@…>

Branches:

master-v0.7, master-v0.6, upstream_count_limit

Children:

c79fa6

Parents:

35dacc (diff), dbd2c3 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

git-author:

Tomash Brechko <tomash.brechko@…> (02/04/08 16:12:02)

git-committer:

Tomash Brechko <tomash.brechko@…> (02/04/08 16:12:02)

Message:

Merge branch 'memcached_gzip'

Files:

• memcached_hash/Changes (added)
• memcached_hash/README (added)
• memcached_hash/TODO (added)
• memcached_hash/config (added)
• memcached_hash/dist.sh (added)
• memcached_hash/nginx.conf (added)
• memcached_hash/ngx_http_upstream_memcached_hash_module.c (added)
• memcached_hash/populate.pl (added)
• server/src/core/nginx.c (modified) (1 diff)
• server/src/core/ngx_crc32.c (modified) (1 diff)
• server/src/core/ngx_crc32.h (modified) (1 diff)
• server/src/core/ngx_string.h (modified) (1 diff)
• server/src/http/modules/ngx_http_fastcgi_module.c (modified) (1 diff)
• server/src/http/modules/ngx_http_gzip_filter_module.c (modified) (8 diffs)
• server/src/http/modules/ngx_http_proxy_module.c (modified) (1 diff)
• server/src/http/ngx_http_upstream.c (modified) (4 diffs)
• server/src/http/ngx_http_upstream.h (modified) (2 diffs)

server/src/core/nginx.c

@@ -274 +274 @@
     }
 
-    /* ngx_crc32_init() requires ngx_cacheline_size set in ngx_os_init() */
-
-    if (ngx_crc32_init() != NGX_OK) {
+    /*
+     * ngx_crc32_table_init() requires ngx_cacheline_size set in ngx_os_init()
+     */
+
+    if (ngx_crc32_table_init() != NGX_OK) {
         return 1;
     }

server/src/core/ngx_crc32.c

@@ -103 +103 @@
 
 ngx_int_t
-ngx_crc32_init(void)
+ngx_crc32_table_init(void)
 {
     void  *p;

server/src/core/ngx_crc32.h

@@ -50 +50 @@
 
 
-ngx_int_t ngx_crc32_init(void);
+#define ngx_crc32_init(crc)                                                   \
+    crc = 0xffffffff
+
+
+static ngx_inline void
+ngx_crc32_update(uint32_t *crc, u_char *p, size_t len)
+{
+    uint32_t  c;
+
+    c = *crc;
+
+    while (len--) {
+        c = ngx_crc32_table256[(c ^ *p++) & 0xff] ^ (c >> 8);
+    }
+
+    *crc = c;
+}
+
+
+#define ngx_crc32_final(crc)                                                  \
+    crc ^= 0xffffffff
+
+
+ngx_int_t ngx_crc32_table_init(void);
 
 

server/src/core/ngx_string.h

@@ -64 +64 @@
 #define ngx_memset(buf, c, n)     (void) memset(buf, c, n)
 
+#define ngx_memmove(dst, src, n)  (void) memmove(dst, src, n)
 
 #if (NGX_MEMCPY_LIMIT)

server/src/http/modules/ngx_http_fastcgi_module.c

@@ -180 +180 @@
     { ngx_string("invalid_header"), NGX_HTTP_UPSTREAM_FT_INVALID_HEADER },
     { ngx_string("http_500"), NGX_HTTP_UPSTREAM_FT_HTTP_500 },
+    { ngx_string("http_502"), NGX_HTTP_UPSTREAM_FT_HTTP_502 },
     { ngx_string("http_503"), NGX_HTTP_UPSTREAM_FT_HTTP_503 },
+    { ngx_string("http_504"), NGX_HTTP_UPSTREAM_FT_HTTP_504 },
+    { ngx_string("http_507"), NGX_HTTP_UPSTREAM_FT_HTTP_507 },
     { ngx_string("http_404"), NGX_HTTP_UPSTREAM_FT_HTTP_404 },
     { ngx_string("off"), NGX_HTTP_UPSTREAM_FT_OFF },

server/src/http/modules/ngx_http_gzip_filter_module.c

@@ -309 +309 @@
     ngx_http_gzip_ctx_t   *ctx;
     ngx_http_gzip_conf_t  *conf;
+    const char            *method = (r->gunzip ? "inflate" : "deflate");
 
     ctx = ngx_http_get_module_ctx(r, ngx_http_gzip_filter_module);
@@ -366 +367 @@
         if (rc != Z_OK) {
             ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
-                          "deflateInit2() failed: %d", rc);
+                          "%sInit2() failed: %d", method, rc);
             ngx_http_gzip_error(ctx);
             return NGX_ERROR;
@@ -456 +457 @@
 
                 if (ctx->in_buf->last_buf) {
-                    ctx->flush = Z_FINISH;
+                    /*
+                     * We use Z_BLOCK below for the following reason:
+                     * if we would use Z_FINISH, then decompression
+                     * may return Z_BUF_ERROR, meaning there wasn't
+                     * enough room for decompressed data.  This error
+                     * is not fatal according to zlib.h, however
+                     * ignoring it is dangerous and may mask real
+                     * bugs.  For instance, sometimes completely empty
+                     * last buffer is passed to this filter, and
+                     * decompression would enter the infinite loop: no
+                     * progress is possible because input is void and
+                     * Z_BUF_ERROR is ignored.  We can't use
+                     * Z_NO_FLUSH, because it will never return
+                     * Z_STREAM_END.  We can't use Z_SYNC_FLUSH,
+                     * because it has a special meaning.  So we use
+                     * Z_BLOCK, which eventually would return
+                     * Z_STREAM_END.
+                     *
+                     * Perhaps the above is also true for compression,
+                     * but right now we won't try to change the old
+                     * behaviour.
+                     */
+                    ctx->flush = (!r->gunzip ? Z_FINISH : Z_BLOCK);
 
                 } else if (ctx->in_buf->flush) {
@@ -474 +497 @@
 
 
-            /* is there a space for the gzipped data ? */
+            /* is there a space for the output data ? */
 
             if (ctx->zstream.avail_out == 0) {
@@ -503 +526 @@
             }
 
-            ngx_log_debug6(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                         "deflate in: ni:%p no:%p ai:%ud ao:%ud fl:%d redo:%d",
+            ngx_log_debug7(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                         "%s in: ni:%p no:%p ai:%ud ao:%ud fl:%d redo:%d",
+                         method,
                          ctx->zstream.next_in, ctx->zstream.next_out,
                          ctx->zstream.avail_in, ctx->zstream.avail_out,
@@ -515 +539 @@
             }
 
-            if (rc != Z_OK && rc != Z_STREAM_END && rc != Z_BUF_ERROR) {
+            if (rc != Z_OK && rc != Z_STREAM_END) {
                 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
-                              "deflate() failed: %d, %d", ctx->flush, rc);
+                              "%s() failed: %d, %d", method, ctx->flush, rc);
                 ngx_http_gzip_error(ctx);
                 return NGX_ERROR;
             }
 
-            ngx_log_debug5(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                           "deflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d",
+            ngx_log_debug6(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                           "%s out: ni:%p no:%p ai:%ud ao:%ud rc:%d",
+                           method,
                            ctx->zstream.next_in, ctx->zstream.next_out,
                            ctx->zstream.avail_in, ctx->zstream.avail_out,
                            rc);
 
-            ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                           "gzip in_buf:%p pos:%p",
-                           ctx->in_buf, ctx->in_buf->pos);
+            ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                           "%s in_buf:%p pos:%p",
+                           method, ctx->in_buf, ctx->in_buf->pos);
 
 
@@ -588 +613 @@
                 if (rc != Z_OK) {
                     ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
-                                  "deflateEnd() failed: %d", rc);
+                                  "%sEnd() failed: %d", method, rc);
                     ngx_http_gzip_error(ctx);
                     return NGX_ERROR;
@@ -601 +626 @@
                 }
 
-                cl->buf = ctx->out_buf;
-                cl->next = NULL;
-                *ctx->last_out = cl;
-                ctx->last_out = &cl->next;
+                /*
+                 * On decompression we could already output everything
+                 * under (ctx->flush == Z_SYNC_FLUSH), so we test here
+                 * that the buffer is not empty.
+                 */
+                if (!r->gunzip || ctx->out_buf->last > ctx->out_buf->pos) {
+                    cl->buf = ctx->out_buf;
+                    cl->next = NULL;
+                    *ctx->last_out = cl;
+                    ctx->last_out = &cl->next;
+                }
 
                 if (!r->gunzip) {

server/src/http/modules/ngx_http_proxy_module.c

@@ -134 +134 @@
     { ngx_string("invalid_header"), NGX_HTTP_UPSTREAM_FT_INVALID_HEADER },
     { ngx_string("http_500"), NGX_HTTP_UPSTREAM_FT_HTTP_500 },
+    { ngx_string("http_502"), NGX_HTTP_UPSTREAM_FT_HTTP_502 },
     { ngx_string("http_503"), NGX_HTTP_UPSTREAM_FT_HTTP_503 },
+    { ngx_string("http_504"), NGX_HTTP_UPSTREAM_FT_HTTP_504 },
+    { ngx_string("http_507"), NGX_HTTP_UPSTREAM_FT_HTTP_507 },
     { ngx_string("http_404"), NGX_HTTP_UPSTREAM_FT_HTTP_404 },
     { ngx_string("off"), NGX_HTTP_UPSTREAM_FT_OFF },

server/src/http/ngx_http_upstream.c

@@ -1094 +1094 @@
     }
 
+    if (u->headers_in.status_n == NGX_HTTP_BAD_GATEWAY) {
+
+        if (u->peer.tries > 1
+            && u->conf->next_upstream & NGX_HTTP_UPSTREAM_FT_HTTP_502)
+        {
+            ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_HTTP_502);
+            return;
+        }
+    }
+
+    if (u->headers_in.status_n == NGX_HTTP_SERVICE_UNAVAILABLE) {
+
+        if (u->peer.tries > 1
+            && u->conf->next_upstream & NGX_HTTP_UPSTREAM_FT_HTTP_503)
+        {
+            ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_HTTP_503);
+            return;
+        }
+    }
+
+    if (u->headers_in.status_n == NGX_HTTP_GATEWAY_TIME_OUT) {
+
+        if (u->peer.tries > 1
+            && u->conf->next_upstream & NGX_HTTP_UPSTREAM_FT_HTTP_504)
+        {
+            ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_HTTP_504);
+            return;
+        }
+    }
+
+    if (u->headers_in.status_n == NGX_HTTP_INSUFFICIENT_STORAGE) {
+
+        if (u->peer.tries > 1
+            && u->conf->next_upstream & NGX_HTTP_UPSTREAM_FT_HTTP_507)
+        {
+            ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_HTTP_507);
+            return;
+        }
+    }
 
     if (u->headers_in.status_n >= NGX_HTTP_BAD_REQUEST
@@ -2173 +2212 @@
         switch(ft_type) {
 
+        case NGX_HTTP_UPSTREAM_FT_HTTP_504:
         case NGX_HTTP_UPSTREAM_FT_TIMEOUT:
             status = NGX_HTTP_GATEWAY_TIME_OUT;
@@ -2183 +2223 @@
         case NGX_HTTP_UPSTREAM_FT_HTTP_404:
             status = NGX_HTTP_NOT_FOUND;
+            break;
+
+        case NGX_HTTP_UPSTREAM_FT_HTTP_502:
+            status = NGX_HTTP_BAD_GATEWAY;
+            break;
+
+        case NGX_HTTP_UPSTREAM_FT_HTTP_503:
+            status = NGX_HTTP_SERVICE_UNAVAILABLE;
+            break;
+
+        case NGX_HTTP_UPSTREAM_FT_HTTP_507:
+            status = NGX_HTTP_INSUFFICIENT_STORAGE;
             break;
 
@@ -3139 +3191 @@
     }
 
+    us->name = u.url;
     us->addrs = u.addrs;
     us->naddrs = u.naddrs;

server/src/http/ngx_http_upstream.h

@@ -25 +25 @@
 #define NGX_HTTP_UPSTREAM_FT_BUSY_LOCK       0x00000080
 #define NGX_HTTP_UPSTREAM_FT_MAX_WAITING     0x00000100
+#define NGX_HTTP_UPSTREAM_FT_HTTP_502        0x00000200
+#define NGX_HTTP_UPSTREAM_FT_HTTP_504        0x00000400
+#define NGX_HTTP_UPSTREAM_FT_HTTP_507        0x00000800
 #define NGX_HTTP_UPSTREAM_FT_OFF             0x80000000
 
@@ -65 +68 @@
 
 typedef struct {
+    ngx_str_t                       name;
     ngx_peer_addr_t                *addrs;
     ngx_uint_t                      naddrs;