
P'^c           @@  s/  d  Z  d d l m Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l m	 Z	 d d l
 m Z d d l m Z d d l m Z d d l m Z d d	 l m Z d d
 l m Z d d l m Z d d l m Z m Z d d l m Z d d l m Z d d l m  Z  m! Z! d d l" m# Z# d d l$ m% Z% d d l$ m& Z& d d l' m( Z( d d l) m* Z* d d l+ m, Z, d d l- m. Z. m/ Z/ d d l0 m1 Z1 d d l2 m3 Z3 d d l4 m5 Z5 d d l6 m7 Z7 d d l8 m9 Z9 m: Z: d d l; m< Z< m= Z= m> Z> d d l? m@ Z@ mA ZA mB ZB mC ZC y
 eD ZE Wn' eF k
 r[d eG f d      YZE n Xd! d" d# d$ d% d& d' d( d) d* d+ g ZH e& jI ZI e3 eJ  ZK eK jL eK jM eK jN eK jO eK jP f \ ZL ZM ZQ ZO ZR d, ZS d- ZT d. ZU d/ ZV d0 ZW d1 ZX d2 ZY d3 ZZ d4 Z[ e d5 d6  Z\ d7   Z] d! eG f d8     YZ^ d9   Z_ d:   Z` d;   Za d" e& jb f d<     YZc d# e& jb f d=     YZd d$ e& jb f d>     YZe d) e& jb f d?     YZf d& e& jb f d@     YZg d( e& jb f dA     YZh d% e& jb f dB     YZi d* e& jj f dC     YZk d' e& jb f dD     YZl d S(E   s   
celery.worker.consumer
~~~~~~~~~~~~~~~~~~~~~~

This module contains the components responsible for consuming messages
from the broker, processing the messages and keeping the broker connections
up and running.

i    (   t   absolute_importN(   t   defaultdict(   t   partial(   t   heappush(   t
   itemgetter(   t   sleep(   t   restart_state(   t   RestartFreqExceeded(   t	   DummyLock(   t   QoSt   ignore_errors(   t   _detect_environment(   t	   get_errno(   t	   safe_reprt   bytes_t(   t   TokenBucket(   t   chain(   t	   bootsteps(   t   build_tracer(   t	   signature(   t   InvalidTaskError(   t   itemst   values(   t   noop(   t
   get_logger(   t   Bunch(   t   truncate(   t   humanize_secondst   ratei   (   t	   heartbeatt   loopst   pidbox(   t   task_reservedt   maybe_shutdownt   revokedt   reserved_requestst   buffer_tc           B@  s   e  Z RS(    (   t   __name__t
   __module__(    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR$   6   s   t   Consumert
   Connectiont   Eventst   Heartt   Controlt   Taskst   Evloopt   Agentt   Minglet   Gossipt	   dump_bodysM   consumer: Connection to broker lost. Trying to re-establish the connection...s   Trying again {when}...s'   consumer: Cannot connect to %s: %s.
%s
s   Will retry using next failover.sj   Received and deleted unknown message. Wrong destination?!?

The full contents of the message body was: %s
s  Received unregistered task of type %s.
The message has been ignored and discarded.

Did you remember to import the module containing this task?
Or maybe you are using relative imports?
Please see http://bit.ly/gLye1c for more information.

The full contents of the message body was:
%s
s   Received invalid task message: %s
The message has been ignored and discarded.

Please ensure your message conforms to the task
message protocol as described here: http://bit.ly/hYj41y

The full contents of the message body was:
%s
sI   Can't decode message body: %r [type:%r encoding:%r headers:%s]

body: %s
sT   body: {0}
{{content_type:{1} content_encoding:{2}
  delivery_info:{3} headers={4}}}
t   clockR"   c         C@  sF   t  | t  r t |  } n  d j t t |  d  t |  j   S(   Ns
   {0} ({1}b)i   (   t
   isinstanceR$   R   t   formatR   R   t   lent   body(   t   mR6   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR1      s    c           B@  sV  e  Z e Z e Z d Z d Z d Z	 d  Z
 d e j f d     YZ e d d d d d d d d e d d d  Z d   Z d   Z d d	  Z d
   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d d d d  Z d   Z d   Z d   Z  d   Z! d   Z" d   Z# d   Z$ d   Z% d   Z& RS(    it	   Blueprintc        	   B@  s8   e  Z d  Z d d d d d d d d d	 g	 Z d
   Z RS(   R'   s!   celery.worker.consumer:Connections   celery.worker.consumer:Mingles   celery.worker.consumer:Eventss   celery.worker.consumer:Gossips   celery.worker.consumer:Hearts   celery.worker.consumer:Controls   celery.worker.consumer:Taskss   celery.worker.consumer:Evloops   celery.worker.consumer:Agentc         C@  s   |  j  | d  d  S(   Nt   shutdown(   t   send_all(   t   selft   parent(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR9      s    (   R%   R&   t   namet   default_stepsR9   (    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR8      s   	i   i   c         K@  s  | |  _  | |  _ | |  _ | p* t j   |  _ t j   |  _ | |  _	 | |  _
 |  j   |  _ |  j  j   } | j |  _ | j |  _ t d d d d  |  _ t j t j  |  _ | |  _ t   |  _ |  j  j j |  _ | |  _ | |  _ | |  _ t  d    |  _! |  j"   | |  _# |  j# rT|	 |  _$ |  j$ d  k r]|  j  j j& |  _$ q]n	 d |  _$ t' |  d  s| r{t( j) n t( j* |  _+ n  t,   d k rd  |  j  j _- n  g  |  _. |  j/ d	 |  j  d
 |  j0  |  _1 |  j1 j2 |  t3 |
 pi  |   d  S(   Nt   maxRi   t   maxTi   c           S@  s   d  S(   N(   t   None(    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   <lambda>   t    i    t   loopt   geventt   appt   on_close(4   RF   t
   controllert   init_callbackt   sockett   gethostnamet   hostnamet   ost   getpidt   pidt   poolt   timert
   Strategiest
   strategiest
   connectiont   connection_errorst   channel_errorsR   t   _restart_statet   loggert   isEnabledFort   loggingt   INFOt
   _does_infot   on_task_requestt   sett   on_task_messaget   conft   BROKER_HEARTBEAT_CHECKRATEt   amqheartbeat_ratet   disable_rate_limitst   initial_prefetch_countt   prefetch_multiplierR   t   task_bucketst   reset_rate_limitst   hubt   amqheartbeatRA   t   BROKER_HEARTBEATt   hasattrR   t   asynloopt   synloopRD   R   t   BROKER_CONNECTION_TIMEOUTt   stepsR8   RG   t	   blueprintt   applyt   dict(   R;   R]   RI   RL   RP   RF   RQ   RH   Rh   Ri   t   worker_optionsRc   Rd   Re   t   kwargst   conninfo(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   __init__   sF    									
						c         C@  s2   t  t | d d    } | r. t | d d Sd  S(   Nt
   rate_limitt   capacityi   (   R   t   getattrRA   R   (   R;   t   typet   limit(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   bucket_for_task   s    c         @  s0     j  j   f d   t   j j  D  d  S(   Nc         3@  s*   |  ]  \ } } |   j  |  f Vq d  S(   N(   R|   (   t   .0t   nt   t(   R;   (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pys	   <genexpr>   s    (   Rf   t   updateR   RF   t   tasks(   R;   (    (   R;   s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRg      s    	i    c         C@  sD   |  j  j } |  j s | r! d S|  j  j |  j |  _ |  j |  S(   sy  Update prefetch count after pool/shrink grow operations.

        Index must be the change in number of processes as a positive
        (increasing) or negative (decreasing) number.

        .. note::

            Currently pool grow operations will end up with an offset
            of +1 if the initial size of the pool was 0 (e.g.
            ``--autoscale=1,0``).

        N(   RP   t   num_processesRd   Re   t   _update_qos_eventually(   R;   t   indexR   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   _update_prefetch_count   s
    c         C@  s5   | d k  r |  j  j n	 |  j  j t |  |  j  S(   Ni    (   t   qost   decrement_eventuallyt   increment_eventuallyt   absRe   (   R;   R   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s    	c         C@  s^   | j  |  sC | j |  } |  j j | |  j | | | f  n t |  |  j |  d  S(   N(   t   can_consumet   expected_timeRQ   t
   call_aftert   _limit_taskR    R]   (   R;   t   requestt   buckett   tokenst   hold(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s    	
c         C@  s?  |  j  } x/| j t k r:|  j d 7_ t   y | j |   Wq |  j k
 r6} t | t  r t	 |  t
 j k r   n  t   y |  j j   Wn0 t k
 r } t d | d d t d  n X| j t k r7|  j r7t t d t y |  j j   Wn t k
 rn X|  j   | j |   q7q Xq Wd  S(   Ni   s   Frequent restarts detected: %rt   exc_info(   Rp   t   statet   CLOSEt   restart_countR!   t   startRU   R3   t   OSErrorR   t   errnot   EMFILERW   t   stepR   t   critR   RT   t   warnt   CONNECTION_RETRYt   Truet   collectt	   ExceptionRG   t   restart(   R;   Rp   t   exc(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s.    	$
c         C@  s&   |  j  j |  d d | f d d d  S(   Nt   register_with_event_loopt   argst   descriptions   Hub.register(   Rp   R:   (   R;   Rh   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR   +  s    	c         C@  s   t  |  _ |  j j |   d  S(   N(   R   t   in_shutdownRp   R9   (   R;   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR9   1  s    	c         C@  s   |  j  j |   d  S(   N(   Rp   t   stop(   R;   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR   5  s    c         C@  s*   |  j  d  } |  _  | r& | |   n  d  S(   N(   RI   RA   (   R;   t   callback(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   on_ready8  s    c      	   C@  s:   |  |  j  |  j |  j |  j |  j |  j |  j j |  j f	 S(   N(	   RT   t   task_consumerRp   Rh   R   Ri   RF   R2   Rb   (   R;   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt	   loop_args=  s    c      	   C@  sH   t  t | | j | j t | j  t | | j  d d | j   d S(   s  Callback called if an error occurs while decoding
        a message received.

        Simply logs the error and acknowledges the message so it
        doesn't enter a loop.

        :param message: The message with errors.
        :param exc: The original exception instance.

        R   i   N(	   R   t   MESSAGE_DECODE_ERRORt   content_typet   content_encodingR   t   headersR1   R6   t   ack(   R;   t   messageR   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   on_decode_errorB  s
    c         C@  st   |  j  r( |  j  j r( |  j  j j   n  |  j rA |  j j   n  t j   |  j rp |  j j rp |  j j   n  d  S(   N(   RH   t	   semaphoret   clearRQ   R#   RP   t   flush(   R;   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRG   S  s    	
c         @  s   |  j  j d |  j    t   f d  } |  j  j j sG   j     S  j | |  j  j j d t	   |  j
 r   j j   j |  j
  n    S(   s   Establish the broker connection.

        Will retry establishing the connection if the
        :setting:`BROKER_CONNECTION_RETRY` setting is enabled

        R   c      
   @  s\   t    d d   r' | d k r' t } n  t t   j   |  | j d t | d d    d  S(   Nt   alti    t   whent   int    (   Ry   RA   t   CONNECTION_FAILOVERt   errort   CONNECTION_ERRORt   as_uriR4   R   (   R   t   intervalt	   next_step(   t   conn(    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   _error_handlerj  s    	R   (   RF   RT   Ri   t   CONNECTION_RETRY_STEPR`   t   BROKER_CONNECTION_RETRYt   connectt   ensure_connectiont   BROKER_CONNECTION_MAX_RETRIESR!   Rh   t	   transportR   (   R;   R   (    (   R   s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR   _  s    
		c   	   	   K@  s   |  j  } |  j j j } | | k r1 | | } nT | d  k rC | n | } | d  k r[ d n | } | j | d | d | d | | } | j |  s | j |  | j   t	 d |  n  d  S(   Nt   directt   exchanget   exchange_typet   routing_keys   Started consuming from %s(
   R   RF   t   amqpt   queuesRA   t
   select_addt   consuming_fromt	   add_queuet   consumet   info(	   R;   t   queueR   R   R   t   optionst   csetR   t   q(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   add_task_queue  s    	
c         C@  s7   t  d |  |  j j j j |  |  j j |  d  S(   Ns   Canceling queue %s(   R   RF   R   R   t   deselectR   t   cancel_by_queue(   R;   R   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   cancel_task_queue  s    c         C@  s(   t  |  |  j |  |  j j   d S(   sI   Method called by the timer to apply a task with an
        ETA/countdown.N(   R    R]   R   R   (   R;   t   task(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   apply_eta_task  s    
c         C@  sF   t  j t | |  t | j  t | j  t | j  t | j   S(   N(   t   MESSAGE_REPORTR4   R1   R   R   R   t   delivery_infoR   (   R;   R6   R   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   _message_report  s
    c         C@  s0   t  t |  j | |   | j t |  j  d  S(   N(   R   t   UNKNOWN_FORMATR   t   reject_log_errorRX   RU   (   R;   R6   R   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   on_unknown_message  s    c         C@  s6   t  t | t | |  d t | j t |  j  d  S(   NR   (   R   t   UNKNOWN_TASK_ERRORR1   R   R   RX   RU   (   R;   R6   R   R   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   on_unknown_task  s    c         C@  s6   t  t | t | |  d t | j t |  j  d  S(   NR   (   R   t   INVALID_TASK_ERRORR1   R   R   RX   RU   (   R;   R6   R   R   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   on_invalid_task  s    c         C@  ss   |  j  j } x` t |  j  j  D]L \ } } | j |  j  |   |  j | <t | | | |  j d |  j  | _ q Wd  S(   NRF   (	   RF   t   loaderR   R   t   start_strategyRS   R   RL   t	   __trace__(   R;   R   R=   R   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   update_strategies  s
    c         @  sO    j    j   j   j   j          f d   } | S(   Nc         @  s  | j  } y | d d } } WnP t t f k
 rp y |  d d } } Wqq t t f k
 rl  |  |  SXn X| r t  j | |  | |  }  n  y$  | | |  | j | j    WnE t k
 r }  |  | |  n# t k
 r }  |  | |  n Xd  S(   NR   i   i    (   R   t   KeyErrort	   TypeErrort   proto2_to_proto1RF   t   ack_log_errorR   R   (   R6   R   R   t   type_t	   is_proto2R   (   t	   callbacksR   R   R   R;   RS   (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   on_task_received  s(    	(   RS   R   R   R   R_   (   R;   R   (    (   R   R   R   R   R;   RS   s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   create_task_handler  s    					c         C@  s   d j  d |  d |  j j    S(   Ns%   <Consumer: {self.hostname} ({state})>R;   R   (   R4   Rp   t   human_state(   R;   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   __repr__  s    	N('   R%   R&   Rr   RR   t   FalseR   RA   RI   RP   RQ   R   R   R8   R   Rv   R|   Rg   R   R   R   R   R   R9   R   R   R   R   RG   R   R   R   R   R   R   R   R   R   R   R   (    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR'      sJ   2				
									 								!c         C@  sx   | \ } } } t  |   } | j d  }	 t t | |  d | d | | }
 |	 rt |
 d j t |	 d |   n  |
 S(   NR   R   Rt   R   RF   (   t   _extract_proto2_embedt   popRr   t   _extract_proto2_headerst   appendR   (   RF   R   R6   R   R   R   Rt   t   embedt   embeddedt   chainedt   new_body(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s    	 c         K@  s<   i | d 6|  d 6| d 6| d 6| d 6t  d 6| d 6| d 6S(	   Nt   idR   t   retriest   etat   expirest   utct   tasksett	   timelimit(   R   (   R   R   R   R  R  t   groupR  t   _(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s    c         K@  s&   i |  p g  d 6| d 6| d 6| d 6S(   NR   t   errbacksR   t   chord(    (   R   R  R   R	  R  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s
    c           B@  s/   e  Z d    Z d   Z d   Z d d  Z RS(   c         K@  s   d  | _ d  S(   N(   RA   RT   (   R;   t   cRt   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRv   	  s    c         C@  s)   | j    | _ t d | j j    d  S(   Ns   Connected to %s(   R   RT   R   R   (   R;   R
  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s    c         C@  s0   | j  d  } | _  | r, t | | j  n  d  S(   N(   RT   RA   R
   t   close(   R;   R
  RT   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR9     s    s   N/Ac         C@  s6   | j  r+ | j  j   } | j d d   n  i | d 6S(   Nt   passwordt   broker(   RT   R   R   RA   (   R;   R
  t   params(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s    	(   R%   R&   Rv   R   R9   R   (    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR(     s   			c           B@  sA   e  Z e f Z d d   Z d   Z d   Z d   Z d   Z	 RS(   c         K@  s.   t  |  _ | r d  n d g |  _ d  | _ d  S(   Nt   worker(   R   t   send_eventsRA   t   groupst   event_dispatcher(   R;   R
  R  Rt   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRv   !  s    	c         C@  sp   |  j  |  } | j j j | j   d | j d |  j d |  j } | _ | rl | j	 |  | j
   n  d  S(   NRL   t   enabledR  (   t   _closeRF   t   eventst
   DispatcherR   RL   R  R  R  t   extend_bufferR   (   R;   R
  t   prevt   dis(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR   &  s    c         C@  s   d  S(   N(    (   R;   R
  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR   1  s    c         C@  s^   | j  rZ | j  } | j |  _ | j r= t | | j j  n  t | | j  d  | _  | Sd  S(   N(   R  R  RT   R
   R  RA   (   R;   R
  t
   dispatcher(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR  4  s    				c         C@  s   |  j  |  d  S(   N(   R  (   R;   R
  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR9   A  s    N(
   R%   R&   R(   t   requiresRA   Rv   R   R   R  R9   (    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR)     s   				c           B@  s8   e  Z e f Z e d d   Z d   Z d   Z e Z	 RS(   c         K@  s    | |  _  | |  _ d  | _ d  S(   N(   R  t   heartbeat_intervalRA   t   heart(   R;   R
  t   without_heartbeatR  Rt   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRv   H  s    
	c         C@  s2   t  j | j | j |  j  | _ | j j   d  S(   N(   R   R*   RQ   R  R  R  R   (   R;   R
  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR   N  s    c         C@  s   | j  o | j  j   | _  d  S(   N(   R  R   (   R;   R
  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR   T  s    N(
   R%   R&   R)   R  R   RA   Rv   R   R   R9   (    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR*   E  s
   			c           B@  sG   e  Z d  Z e f Z e d d g  Z e d  Z d   Z	 d   Z
 RS(   R/   R   t   redisc         K@  s    | o |  j  | j  |  _ d  S(   N(   t   compatible_transportRF   R  (   R;   R
  t   without_mingleRt   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRv   ^  s    c         C@  s,   | j     } | j j |  j k SWd  QXd  S(   N(   RT   R   t   driver_typet   compatible_transports(   R;   RF   R   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR   a  s    c         C@  s  t  d  | j j j d d d | j  } | j | j t j  pF i  } | j	 | j d   | rt  d t g  t |  D] \ } } | rx | ^ qx   xg t |  D]Y } | r y t |  \ } } Wn t k
 r q X| j j j |  t j |  q q Wt  d  n
 t  d  d  S(   Ns   mingle: searching for neighborst   timeoutg      ?RT   s   mingle: sync with %s nodess   mingle: sync completes   mingle: all alone(   R   RF   t   controlt   inspectRT   t   helloRL   R"   t   _dataR   RA   R5   R   R   t   MINGLE_GET_FIELDSR   R2   t   adjustR   (   R;   R
  t   It   repliest   replyt   valuet   other_clockt   other_revoked(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR   e  s"    
!2(   R%   R&   t   labelR)   R  R^   R#  R   Rv   R   R   (    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR/   Y  s   		c           B@  s>   e  Z e f Z d    Z d   Z d   Z d   Z d   Z RS(   c         K@  s   d  | _ | _ d  S(   N(   RA   R   R   (   R;   R
  Rt   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRv   ~  s    c         @  s     j      j j    j j j d   j     j j j   j d   j	   _
    f d   } t |   j    _ d  S(   Ni    R   c         @  s     j  j d |  d   S(   Nt   prefetch_countt   apply_global(   R   R   (   R2  (   R
  t
   qos_global(    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   set_prefetch_count  s    (   R   RT   t   qos_semantics_matches_spect   default_channelt	   basic_qosRd   RF   R   t   TaskConsumerR   R   R	   R   (   R;   R
  R5  (    (   R
  R4  s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s    
c         C@  s-   | j  r) t d  t | | j  j  n  d  S(   Ns   Canceling task consumer...(   R   t   debugR
   t   cancel(   R;   R
  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s    	
c         C@  sC   | j  r? |  j |  t d  t | | j  j  d  | _  n  d  S(   Ns   Closing consumer channel...(   R   R   R:  R
   R  RA   (   R;   R
  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR9     s
    	
c         C@  s    i | j  r | j  j n d d 6S(   Ns   N/AR2  (   R   R.  (   R;   R
  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s    (	   R%   R&   R/   R  Rv   R   R   R9   R   (    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR,   {  s   					c           B@  s)   e  Z e Z e f Z d    Z d   Z RS(   c         K@  s   | j  j j |  _ |  _ d  S(   N(   RF   R`   t   CELERYD_AGENTt	   agent_clsR  (   R;   R
  Rt   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRv     s    c         C@  s#   |  j  |  j | j  } | _ | S(   N(   t   instantiateR=  RT   t   agent(   R;   R
  R?  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   create  s    (   R%   R&   R   t   conditionalR(   R  Rv   R@  (    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR.     s   		c           B@  s#   e  Z e f Z d    Z d   Z RS(   c         K@  ss   | j  d  k	 o | j  j |  _ |  j r0 t j n t j |  |  _ |  j j |  _ |  j j |  _ |  j j	 |  _	 d  S(   N(
   RP   RA   t   is_greenR   t   gPidboxt   Pidboxt   boxR   R   R9   (   R;   R
  Rt   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRv     s
    $c         C@  s   | j  j j S(   N(   RF   R`   t   CELERY_ENABLE_REMOTE_CONTROL(   R;   R
  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt
   include_if  s    (   R%   R&   R,   R  Rv   RG  (    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR+     s   		c           B@  s   e  Z d  Z e f Z e d d d d d d d  Z e d d	 g  Z e	 d
 d  Z
 d   Z d d  Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z RS(   R0   R   R2   RL   RO   t   topict   actiont   cverR   R  g      @c         K@  ss  | o |  j  | j  |  _ | j |  _ |  | _ | j j j |  _ | j |  _ d j |  j t | j	  g  |  _
 t d t   d t   d t    |  _ | j |  _ |  j r| j j j d |  j d |  j d d  |  _ | j r t   | _ n  |  j j |  _ n  | |  _ d  |  _ t t  |  _ i  |  _ i |  j d	 6|  j  d
 6|  _! | j j" |  _" i |  j# d 6|  _$ d  S(   Nt   .t	   node_joint
   node_leavet	   node_lostt   on_node_joint   on_node_leavet   max_tasks_in_memoryi   s   worker.elects   worker.elect.ackR   (%   R   RF   R  t   gossipR  t   ReceiverRL   t   joint   strRO   t   full_hostnameR   R^   t   onRQ   t   StateRO  RP  R   Rh   R   t   _mutext   eventt   update_stateR   RA   t   _trefR   t   listt   consensus_requestst   consensus_repliest   on_electt   on_elect_ackt   event_handlersR2   t	   call_taskt   election_handlers(   R;   R
  t   without_gossipR   Rt   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRv     s:    	$									
c         C@  s,   | j     } | j j |  j k SWd  QXd  S(   N(   RT   R   R"  R#  (   R;   RF   R   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s    c      
   C@  s9   g  |  j  | <|  j j d d | d | d | d d d  S(   Ns   worker-electR   RH  RI  RJ  i   (   R_  R  t   send(   R;   R   RH  RI  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   election  s    	c         C@  sJ   y t  | d |  j j   Wn& t k
 rE } t d | d d n Xd  S(   NRF   s   Could not call task: %rR   i   (   R   RF   t   apply_asyncR   R   (   R;   R   R   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRc    s    c   
      C@  s   y( |  j  |  \ } } } } } } } Wn# t k
 rM }	 t d |	 d d SXt |  j | | d | | f | | f  |  j j d d | d  S(   Ns!   election request missing field %sR   i   s   %s.%ss   worker-elect-ackR   (   t   _cons_stamp_fieldsR   R   R   R^  R  Rf  (
   R;   RZ  t   id_R2   RL   RO   RH  RI  R  R   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR`    s    (
c         C@  s&   t  t |   j |  | j |  _ d  S(   N(   t   superR0   R   R  R  (   R;   R
  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR     s    c   
      C@  s+  | d } y |  j  | } Wn t k
 r/ d  SX|  j j   } | j | d  t |  t |  k r'|  j j |  j |  \ } } } } | |  j	 k r t
 d |  y |  j | }	 Wn$ t k
 r t d | d d q X|	 |  n t
 d | |  |  j j | d   |  j  j | d   n  d  S(   NR   RL   s   I won the election %rs   Unknown election topic %rR   i   s   node %s elected for %r(   R_  R   R   t   alive_workersR   R5   R2   t	   sort_heapR^  RV  R   Rd  R   R   RA   (
   R;   RZ  R   R,  Rl  R  t   leaderRH  RI  t   handler(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRa    s(    
	c         C@  s*   t  d | j  |  j |  j j |  d  S(   Ns   %s joined the party(   R:  RL   t   _call_handlersRW  RL  (   R;   R  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRO  +  s    c         C@  s*   t  d | j  |  j |  j j |  d  S(   Ns   %s left(   R:  RL   Rp  RW  RM  (   R;   R  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRP  /  s    c         C@  s*   t  d | j  |  j |  j j |  d  S(   Ns   missed heartbeat from %s(   R   RL   Rp  RW  RN  (   R;   R  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   on_node_lost3  s    c         O@  sR   xK | D]C } y | | |   Wq t  k
 rI } t d | | d d q Xq Wd  S(   Ns!   Ignored error from handler %r: %rR   i   (   R   R   (   R;   t   handlersR   Rt   Ro  R   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRp  7  s    c         C@  sA   |  j  d  k	 r |  j  j   n  |  j j |  j |  j  |  _  d  S(   N(   R\  RA   R;  RQ   t   call_repeatedlyR   t   periodic(   R;   (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   register_timer?  s    c         C@  sz   |  j  j } t   } x: t |  D], } | j s" | j |  |  j |  q" q" Wx! | D] } | j | j d   qY Wd  S(   N(
   R   t   workersR^   R   t   alivet   addRq  R   RL   RA   (   R;   Rv  t   dirtyR  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRt  D  s    		c         C@  sV   |  j    |  j | d d } t j | d | j g d t |  j | j  d t g S(   NR   s   worker.#R   t
   on_messaget   no_ack(	   Ru  RS  t   kombuR'   R   R   Rz  t   event_from_messageR   (   R;   t   channelt   ev(    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   get_consumersN  s    
c         C@  s   | j  d } | j d d  d d k r- d  Sy |  j | } Wn t k
 rQ n X| | j  S| j j d  p{ | j d } | |  j k r | | j  \ } } |  j |  n |  j	 j
   d  S(   NR   RK  i   i    R   RL   (   R   t   splitRb  R   t   payloadR   t   getRL   R[  R2   t   forward(   R;   t   prepareR   t   _typeRo  RL   Rz   RZ  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyRz  X  s    N(   R%   R&   R1  R/   R  R   Ri  R^   R#  R   Rv   R   RA   Rg  Rc  R`  R   Ra  RO  RP  Rq  Rp  Ru  Rt  R  Rz  (    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR0     s(   	%											
	
c           B@  s&   e  Z d  Z e Z d   Z d   Z RS(   s
   event loopc         C@  s$   |  j  |  | j | j     d  S(   N(   t	   patch_allRD   R   (   R;   R
  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR   r  s    c         C@  s   t    | j _ d  S(   N(   R   R   RY  (   R;   R
  (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR  v  s    (   R%   R&   R1  R   t   lastR   R  (    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyR-   n  s   	(m   t   __doc__t
   __future__R    R   R|  RZ   RM   RJ   t   collectionsR   t	   functoolsR   t   heapqR   t   operatorR   t   timeR   t   billiard.commonR   t   billiard.exceptionsR   t   kombu.async.semaphoreR   t   kombu.commonR	   R
   t	   kombu.synR   t   kombu.utils.compatR   t   kombu.utils.encodingR   R   t   kombu.utils.limitsR   t   celeryR   R   t   celery.app.traceR   t   celery.canvasR   t   celery.exceptionsR   t   celery.fiveR   R   t   celery.utils.functionalR   t   celery.utils.logR   t   celery.utils.objectsR   t   celery.utils.textR   t   celery.utils.timeutilsR   R   RC   R   R   R   R   R    R!   R"   R#   t   bufferR$   t	   NameErrort   objectt   __all__R   R%   RX   R:  R   t   warningR   t   criticalR   R   R   R   R   R   R   R   R   R   R   R)  R1   R'   R   R   R   t   StartStopStepR(   R)   R*   R/   R,   R.   R+   t   ConsumerStepR0   R-   (    (    (    s8   /tmp/pip-unpacked-wheel-gV1wwp/celery/worker/consumer.pyt   <module>
   s   "
	!	 Y				'".