Shulou(Shulou.com)05/31 Report--

Editor to share with you how nginx dynamically modify upstream ngx_http_dyups_module, I believe that most people do not know much about it, so share this article for your reference, I hope you can learn a lot after reading this article, let's go to know it!

Brief introduction

Nginx dynamically modifies upstream non-reload nginx module, ngx_http_dyups_module analysis.

It is mainly analyzed and recorded around https://github.com/yzprofile/ngx_http_dyups_module/blob/master/ngx_http_dyups_module.c.

Open it up.

Initialize this array on create_main_conf

Static void * ngx_http_dyups_create_main_conf (ngx_conf_t * cf)

{

...

If (& dmcf- > dy_upstreams, cf- > pool, 1024, sizeof (ngx_http_dyups_srv_conf_t))! = NGX_OK)

{

Return NULL

}

...

}

Ngx_http_dyups_init

Static ngx_http_module_t ngx_http_dyups_module_ctx = {

Ngx_http_dyups_pre_conf, / * preconfiguration * /

Ngx_http_dyups_init, / * postconfiguration * /

Ngx_http_dyups_create_main_conf, / * create main configuration * /

Ngx_http_dyups_init_main_conf, / * init main configuration * /

Ngx_http_dyups_create_srv_conf, / * create server configuration * /

NULL, / * merge server configuration * /

NULL, / * create location configuration * /

NULL / * merge location configuration * /

}

When dyups init, take the conf out of the upstream and put it in it.

Initialize the dy_upstream chain and the global ngx_http_dyups_deleted_upstream.

Static ngx_int_t ngx_http_dyups_init (ngx_conf_t * cf)

{

...

Dmcf = ngx_http_conf_get_module_main_conf (cf, ngx_http_dyups_module)

Umcf = ngx_http_conf_get_module_main_conf (cf, ngx_http_upstream_module)

Uscfp = umcf- > upstreams.elts

For (I = 0; I)

< umcf->

Upstreams.nelts; iTunes +) {

Duscf = ngx_array_push (& dmcf- > dy_upstreams)

/ / zeroing

Ngx_memzero (duscf, sizeof (ngx_http_dyups_srv_conf_t))

Duscf- > pool = NULL

/ / assignment

Duscf- > upstream = uscfp [I]

Duscf- > dynamic = (uscfp [I]-> port = = 0

& & uscfp [I]-> srv_conf & & uscfp [I]-> servers

& & uscfp [I]-> flags & NGX_HTTP_UPSTREAM_CREATE)

Duscf- > deleted = 0

/ / assign index

Duscf- > idx = I

}

...

}

Dyups share memory synchronization mechanism

Shm initialization is implemented in the ngx_http_dyups_init_main_conf function, with the read_mesg timeout set and the size specified.

Static char * ngx_http_dyups_init_main_conf (ngx_conf_t * cf, void * conf)

{

...

If (dmcf- > read_msg_timeout = = NGX_CONF_UNSET_MSEC) {

/ / once a second

Dmcf- > read_msg_timeout = 1000

}

If (dmcf- > shm_size = = NGX_CONF_UNSET_UINT) {

Dmcf- > shm_size = 2 * 1024 * 1024

}

Return ngx_http_dyups_init_shm (cf, conf)

...

}

Static char * ngx_http_dyups_init_shm (ngx_conf_t * cf, void * conf)

{

...

Shm_zone = ngx_shared_memory_add (cf, & dmcf- > shm_name, dmcf- > shm_size

& ngx_http_dyups_module)

Shm_zone- > data = cf- > pool

/ / the init function of the shared memory block with this name added will be called uniformly during initialization.

Shm_zone- > init = ngx_http_dyups_init_shm_zone

...

}

Static ngx_int_t ngx_http_dyups_init_shm_zone (ngx_shm_zone_t * shm_zone, void * data)

{

...

Shpool = (ngx_slab_pool_t *) shm_zone- > shm.addr

Sh = ngx_slab_alloc (shpool, sizeof (ngx_dyups_shctx_t))

If (sh = = NULL) {

Return NGX_ERROR

}

/ / Global variables, sh and shpool

Ngx_dyups_global_ctx.sh = sh

Ngx_dyups_global_ctx.shpool = shpool

/ / initialize msg- > queue

Ngx_queue_init (& sh- > msg_queue)

Sh- > version = 0

Sh- > status = NULL

...

}

Ngx_http_dyups_init_process

This function is called when the process is started and some timers are set.

Initialize the shared memory and determine if the exit is abnormal, then reload the upstream configuration.

Static ngx_int_t ngx_http_dyups_init_process (ngx_cycle_t * cycle)

{

...

/ / set a timer to time read msg and synchronize information

Timer = & ngx_dyups_global_ctx.msg_timer

Timer- > handler = ngx_http_dyups_read_msg

Ngx_add_timer (timer, dmcf- > read_msg_timeout)

/ / get the global pool and sh

Shpool = ngx_dyups_global_ctx.shpool

Sh = ngx_dyups_global_ctx.sh

Ngx_shmtx_lock (& shpool- > mutex)

/ / when initializing, it must be the memory of the corresponding number of processes requested by NULL,

If (sh- > status = = NULL) {

Sh- > status = ngx_slab_alloc_locked (shpool

Sizeof (ngx_dyups_status_t) * ccf- > worker_processes)

If (sh- > status = = NULL) {

Ngx_shmtx_unlock (& shpool- > mutex)

Return NGX_ERROR

}

Ngx_memzero (sh- > status)

Sizeof (ngx_dyups_status_t) * ccf- > worker_processes)

Ngx_shmtx_unlock (& shpool- > mutex)

Return NGX_OK

}

Ngx_shmtx_unlock (& shpool- > mutex)

/ / judge version. If it is not 0, it means that version has been + + in synchronization, so the process is hung up and pulled up again.

If (sh- > version! = 0) {

/ /...

}

The core is the ngx_http_dyups_read_msg function, in which is the ngx_http_dyups_read_msg_locked function

Static void ngx_http_dyups_read_msg_locked (ngx_event_t * ev)

{

...

Sh = ngx_dyups_global_ctx.sh

Shpool = ngx_dyups_global_ctx.shpool

For (I = 0; I)

< ccf->

Worker_processes; iTunes +) {

Status = & sh- > status [I]

If (status- > pid = = 0 | | status- > pid = = ngx_pid) {

Ngx_log_debug2 (NGX_LOG_DEBUG_HTTP, ev- > log, 0

"[dyups] process P update time ui"

Status- > pid, status- > time)

/ / traverses all processes and assigns the corresponding pid

Status- > pid = ngx_pid

Status- > time = now

Break

}

}

/ / traversing message queues

For (Q = ngx_queue_last (& sh- > msg_queue)

Q! = ngx_queue_sentinel (& sh- > msg_queue)

Q = ngx_queue_prev (Q))

{

/ / if the count of the msg is the same as the number of processes, then everyone has synchronized. Delete the msg.

If (msg- > count = = ccf- > worker_processes) {

T = ngx_queue_next (Q); ngx_queue_remove (Q); Q = t

Ngx_log_debug2 (NGX_LOG_DEBUG_HTTP, ev- > log, 0, "[dyups] destroy msg% VRV% V", & msg- > name, & msg- > content)

Ngx_dyups_destroy_msg (shpool, msg)

Continue

}

Found = 0

For (I = 0; I)

< msg->

Count; iTunes +) {

Ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, ev- > log, 0, "[dyups] msg pids [% P]", msg- > pid [I])

If (msg- > pid [I] = = ngx_pid) {

Found = 1

Break

}

}

/ / if the process is found, it means it has been synchronized.

If (found) {

Ngx_log_debug2 (NGX_LOG_DEBUG_HTTP, ev- > log, 0, "[dyups] msg% V count% ui found", & msg- > name, msg- > count)

Continue

}

/ / if nothing is found, count++,pid updates

Msg- > pid [I] = ngx_pid

Msg- > count++

Ngx_log_debug2 (NGX_LOG_DEBUG_HTTP, ev- > log, 0, "[dyups] msg% V count% ui", & msg- > name, msg- > count)

/ / take out name and content

Name = msg- > name

Content = msg- > content

/ / perform synchronization

Rc = ngx_dyups_sync_cmd (pool, & name, & content, msg- > flag)

If (rc! = NGX_OK) {

Ngx_log_error (NGX_LOG_ALERT, ev- > log, 0, "[dyups] read msg error, may cause the"config inaccuracy,name:%V, content:%V", & name, & content)

}

}

...

}

Static ngx_int_t ngx_dyups_sync_cmd (ngx_pool_t * pool, ngx_str_t * name, ngx_str_t * content, ngx_uint_t flag)

{

...

} else if (flag = = NGX_DYUPS_ADD) {

Body.start = body.pos = content- > data

Body.end = body.last = content- > data + content- > len

Body.temporary = 1

Rc = ngx_dyups_do_update (name, & body, & rv)

Ngx_log_error (NGX_LOG_INFO, ngx_cycle- > log, 0, "[dyups] sync add:% V rv:% V rc:% I", name, & rv, rc)

If (rc! = NGX_HTTP_OK) {

Return NGX_ERROR

}

Return NGX_OK

}

...

}

Synchronize the information received by other processes, and add it to the message queue if it is processed by the current process.

Ngx_dyups_update_upstream

Ngx_int_t ngx_dyups_update_upstream (ngx_str_t * name, ngx_buf_t * buf, ngx_str_t * rv)

{

...

Ngx_http_dyups_read_msg_locked (timer)

/ / sandbox test configuration

Status = ngx_dyups_sandbox_update (buf, rv)

If (status! = NGX_HTTP_OK) {

Goto finish

}

Status = ngx_dyups_do_update (name, buf, rv)

If (status = = NGX_HTTP_OK) {

/ / send the operation to the queue

If (ngx_http_dyups_send_msg (name, buf, NGX_DYUPS_ADD)) {

Ngx_str_set (rv, "alert: update success"

"but not sync to other process")

Status = NGX_HTTP_INTERNAL_SERVER_ERROR

}

}

...

}

Ngx_http_dyups_send_msg function analysis

Static ngx_int_t ngx_http_dyups_send_msg (ngx_str_t * name, ngx_buf_t * body, ngx_uint_t flag)

{

...

/ / initialize the entire msg and fill in name and body

Sh- > version++

Ngx_queue_insert_head (& sh- > msg_queue, & msg- > queue)

...

}

Ngx_dyups_do_update

Find to find the corresponding upstream before update.

Static ngx_http_dyups_srv_conf_t * ngx_dyups_find_upstream (ngx_str_t * name, ngx_int_t * idx)

{

...

Duscfs = dumcf- > dy_upstreams.elts

For (I = 0; I)

< dumcf->

Dy_upstreams.nelts; iTunes +) {

Duscf = & duscfs [I]

Uscf = duscf- > upstream

If (uscf- > host.len! = name- > len

| | ngx_strncasecmp (uscf- > host.data, name- > data, uscf- > host.len) |

! = 0)

{

Continue

}

* idx = I

Return duscf

}

...

}

If the idx assignment is found.

Once it is found that the dy_upstream of the corresponding name is found, it is judged first.

Then the ngx_dyups_mark_upstream_delete function is called

Static void ngx_dyups_mark_upstream_delete (ngx_http_dyups_srv_conf_t * duscf)

{

...

/ / obtain umcf and uscf

Uscf = duscf- > upstream

Umcf = ngx_http_cycle_get_module_main_conf (ngx_cycle

Ngx_http_upstream_module)

/ / us gets the servers under this dynamic upstream

Us = uscf- > servers- > elts

For (I = 0; I)

< uscf->

Servers- > nelts; iTunes +) {

/ / Mark position 1

Us [I] .down = 1

# if (NGX_HTTP_UPSTREAM_CHECK)

If (us [I] .addrs) {

/ / close peer. See the macro to define the peer that mainly turns off health check.

Ngx_http_upstream_check_delete_dynamic_peer (& uscf- > host

Us [I] .addrs)

}

# endif

}

/ / change the configuration of the index corresponding to upstream into a dummy configuration

Uscfp [duscf-> idx] = & ngx_http_dyups_deleted_upstream

# if (NGX_HTTP_UPSTREAM_RBTREE)

Ngx_rbtree_delete (& umcf- > rbtree, & uscf- > node)

# endif

Duscf- > deleted = NGX_DYUPS_DELETING

...

}

The most important of these is check_delete_dynamic_peer.

Void ngx_http_upstream_check_delete_dynamic_peer (ngx_str_t * name,ngx_addr_t * peer_addr)

{

...

/ * A bunch of comparisons to find choosen*/

Chosen = & per [I]

Chosen- > shm- > ref--

If (chosen- > shm- > ref shm- > delete! = PEER_DELETED) {

Ngx_http_upstream_check_clear_dynamic_peer_shm (chosen- > shm)

Chosen- > shm- > delete = PEER_DELETED

}

Ngx_shmtx_unlock (& chosen- > shm- > mutex)

Ngx_http_upstream_check_clear_peer (chosen)

...

}

After deleting it once and find it again, the probability of idx becomes-1, and you can create it.

Static ngx_int_t ngx_dyups_do_update (ngx_str_t * name, ngx_buf_t * buf, ngx_str_t * rv)

{

...

If (idx =-1) {

Duscf = ngx_array_push (& dumcf- > dy_upstreams)

Uscfp = ngx_array_push (& umcf- > upstreams)

Ngx_memzero (duscf, sizeof (ngx_http_dyups_srv_conf_t))

/ / in order to get the index value of the new upstream in umcf.

Idx = umcf- > upstreams.nelts-1

}

Duscf- > idx = idx

Rc = ngx_dyups_init_upstream (duscf, name, idx)

Rc = ngx_dyups_add_server (duscf, buf)

...

}

The most important are init_upstream and add_server.

The first is init ipstream, whose reference is actually dy_srv_conf_t. The name of upstream and the corresponding index in the upstream linked list

Static ngx_int_t ngx_dyups_init_upstream (ngx_http_dyups_srv_conf_t * duscf, ngx_str_t * name, ngx_uint_t index)

{

...

Umcf = ngx_http_cycle_get_module_main_conf (ngx_cycle

Ngx_http_upstream_module)

Uscfp = umcf- > upstreams.elts

/ * initialize uscf, that is, each structure of upstream * /

Uscfp [index] = uscf; / / assignment

Duscf- > dynamic = 1

Duscf- > upstream = uscf

Ctx = ngx_pcalloc (duscf- > pool, sizeof (ngx_http_conf_ctx_t))

/ / Storage ctx

Duscf- > ctx = ctx

/ / insert go in, uscf

Uscf- > node.key = ngx_crc32_short (uscf- > host.data, uscf- > host.len)

Ngx_rbtree_insert (& umcf- > rbtree, & uscf- > node)

...

}

Static ngx_int_t ngx_dyups_add_server (ngx_http_dyups_srv_conf_t * duscf, ngx_buf_t * buf)

{

...

Ngx_dyups_parse_upstream (& cf, buf)

...

}

Static char * ngx_dyups_parse_upstream (ngx_conf_t * cf, ngx_buf_t * buf)

{

...

B = * buf

Ngx_memzero (& conf_file, sizeof (ngx_conf_file_t))

Conf_file.file.fd = NGX_INVALID_FILE

Conf_file.buffer = & ampb

Cf- > conf_file = & conf_file

Return ngx_conf_parse (cf, NULL)

...

}

Ngx_dyups_do_delete

Static ngx_int_t ngx_dyups_do_delete (ngx_str_t * name, ngx_str_t * rv)

{

...

Duscf = ngx_dyups_find_upstream (name, & dumy)

/ / if NULL is found, or one has been marked to delete, or deleted completely, it means that there is an exception to delete this.

If (duscf = = NULL | | duscf- > deleted) {

Ngx_log_error (NGX_LOG_DEBUG, ngx_cycle- > log, 0, "[dyups] not find upstream% V% p", name, duscf)

Ngx_str_set (rv, "not found uptream")

Return NGX_HTTP_NOT_FOUND

}

/ / if there is no problem, perform a normal deletion

Ngx_dyups_mark_upstream_delete (duscf)

...

}

Ngx_dyups_find_upstream

Find upstream does a lot of things and does some deletions.

Static ngx_http_dyups_srv_conf_t * ngx_dyups_find_upstream (ngx_str_t * name, ngx_int_t * idx)

{

...

Dumcf = ngx_http_cycle_get_module_main_conf (ngx_cycle, ngx_http_dyups_module)

Duscfs = dumcf- > dy_upstreams.elts

For (I = 0; I)

< dumcf->

Dy_upstreams.nelts; iTunes +) {

/ / this is marked in mark_upstream

If (duscf- > deleted = = NGX_DYUPS_DELETING) {

/ / confirm that it can be deleted, depending on the reference count of this ref

If (* (duscf- > ref) = = 0) {

Ngx_log_error (NGX_LOG_INFO, ngx_cycle- > log, 0, "[dyups] free dynamic upstream in find upstream"% ui", duscf- > idx)

Duscf- > deleted = NGX_DYUPS_DELETED

If (duscf- > pool) {

Ngx_destroy_pool (duscf- > pool)

Duscf- > pool = NULL

}

}

}

/ / if it is deleted or deleting, it can't be found unless a deleted is returned after traversing.

If (duscf- > deleted = = NGX_DYUPS_DELETING) {

Continue

}

If (duscf- > deleted = = NGX_DYUPS_DELETED) {

* idx = I

Duscf_del = duscf

Continue

}

/ / return normally if it is found

If (uscf- > host.len! = name- > len

| | ngx_strncasecmp (uscf- > host.data, name- > data, uscf- > host.len) |

! = 0)

{

Continue

}

* idx = I

Return duscf

}

...

}

Reference count ref

The peer init get and free functions of dyups.

Static ngx_int_t ngx_http_dyups_init_peer (ngx_http_request_t * r

Ngx_http_upstream_srv_conf_t * us)

{

...

/ / set context

Ctx = ngx_pcalloc (r-> pool, sizeof (ngx_http_dyups_ctx_t))

If (ctx = = NULL) {

Return NGX_ERROR

}

/ / the data of scf pointing to the corresponding dscf,ctx points to itself.

Ctx- > scf = dscf

Ctx- > data = r-> upstream- > peer.data

Ctx- > get = r-> upstream- > peer.get

Ctx- > free = r-> upstream- > peer.free

R-> upstream- > peer.data = ctx

R-> upstream- > peer.get = ngx_http_dyups_get_peer

R-> upstream- > peer.free = ngx_http_dyups_free_peer

/ / the pool connected to the client registers a destroy function

Cln = ngx_pool_cleanup_add (r-> pool, 0)

If (cln = = NULL) {

Return NGX_ERROR

}

/ / add one to the reference count

Dscf- > ref++

/ / when you call this, you will ref--.

Cln- > handler = ngx_http_dyups_clean_request

Cln- > data = & dscf- > ref

...

}

/ / this function is assigned in the ngx_dyups_add_server initialization upstream

Uscf- > peer.init = ngx_http_dyups_init_peer

/ / the location of the call is here

Static void ngx_http_upstream_init_request (ngx_http_request_t * r)

{

...

If (uscf- > peer.init (r, uscf)! = NGX_OK) {

Ngx_http_upstream_finalize_request (r, u)

NGX_HTTP_INTERNAL_SERVER_ERROR)

Return

}

Ngx_http_upstream_connect (r, u)

...

}

Static ngx_int_t ngx_http_dyups_get_peer (ngx_peer_connection_t * pc, void * data)

{

Ngx_http_dyups_ctx_t * ctx = data

/ / just use the previous peer get.

Return ctx- > get (pc, ctx- > data)

}

Static void ngx_http_dyups_free_peer (ngx_peer_connection_t * pc, void * data

Ngx_uint_t state)

{

Ngx_http_dyups_ctx_t * ctx = data

Ngx_pool_cleanup_t * cln

/ * upstream connect failed * /

If (pc- > connection = = NULL) {

Goto done

}

If (pc- > cached) {

Goto done

}

/ / when free, first call handler to ref++, and then ref--

Ctx- > scf- > ref++

/ / pool sets a data structure that destroys pool and assigns a value to pool- > cleanup

Cln = ngx_pool_cleanup_add (pc- > connection- > pool, 0)

If (cln = = NULL) {

Ngx_log_error (NGX_LOG_ERR, ngx_cycle- > log, 0, "[dyups] dynamic upstream free peer may cause memleak I", ctx- > scf- > ref)

Goto done

}

/ / Recursive function destroyed

Cln- > handler = ngx_http_dyups_clean_request

Cln- > data = & ctx- > scf- > ref

Done:

/ / call the previously saved free after the end

Ctx- > free (pc, ctx- > data, state)

}

Static void ngx_http_dyups_clean_request (void * data)

{

Ngx_uint_t * ref = data

(* ref)

Ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, ngx_cycle- > log, 0, "[dyups] http clean request count% I", * ref)

}

When pool is destroy, the handler is called to subtract the reference count.

Void ngx_destroy_pool (ngx_pool_t * pool)

{

Ngx_pool_t * p, * n

Ngx_pool_large_t * l

Ngx_pool_cleanup_t * c

For (c = pool- > cleanup; c; c = c-> next) {

If (c-> handler) {

Ngx_log_debug1 (NGX_LOG_DEBUG_ALLOC, pool- > log, 0

"run cleanup:% p", c)

C-> handler (c-> data)

}

}

...

}

The above is all the contents of the article "how nginx dynamically modifies upstream ngx_http_dyups_module". Thank you for reading! I believe we all have a certain understanding, hope to share the content to help you, if you want to learn more knowledge, welcome to follow the industry information channel!

Welcome to subscribe "Shulou Technology Information " to get latest news, interesting things and hot topics in the IT industry, and controls the hottest and latest Internet news, technology news and IT industry trends.