File: //usr/share/doc/python3-django-q/html/brokers.html
<!DOCTYPE html>
<html class="writer-html5" lang="en" data-content_root="./">
<head>
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Brokers — Django Q2 1.6.1 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b76e3c8a" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=86f27845" />
<link rel="shortcut icon" href="_static/favicon.ico"/>
<script src="_static/jquery.js?v=8dae8fb0"></script>
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
<script src="_static/documentation_options.js?v=d6a008b6"></script>
<script src="_static/doctools.js?v=888ff710"></script>
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
<script src="_static/js/theme.js"></script>
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="Tasks" href="tasks.html" />
<link rel="prev" title="Configuration" href="configure.html" />
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="index.html" class="icon icon-home">
Django Q2
</a>
<div class="version">
1.6
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="install.html"> Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="configure.html"> Configuration</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#"> Brokers</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#redis">Redis</a></li>
<li class="toctree-l2"><a class="reference internal" href="#ironmq">IronMQ</a></li>
<li class="toctree-l2"><a class="reference internal" href="#amazon-sqs">Amazon SQS</a></li>
<li class="toctree-l2"><a class="reference internal" href="#mongodb">MongoDB</a></li>
<li class="toctree-l2"><a class="reference internal" href="#django-orm">Django ORM</a></li>
<li class="toctree-l2"><a class="reference internal" href="#custom-broker">Custom Broker</a></li>
<li class="toctree-l2"><a class="reference internal" href="#reference">Reference</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#Broker"><code class="docutils literal notranslate"><span class="pre">Broker</span></code></a><ul>
<li class="toctree-l4"><a class="reference internal" href="#Broker.async_task"><code class="docutils literal notranslate"><span class="pre">Broker.async_task()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="#Broker.dequeue"><code class="docutils literal notranslate"><span class="pre">Broker.dequeue()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="#Broker.acknowledge"><code class="docutils literal notranslate"><span class="pre">Broker.acknowledge()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="#Broker.fail"><code class="docutils literal notranslate"><span class="pre">Broker.fail()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="#Broker.delete"><code class="docutils literal notranslate"><span class="pre">Broker.delete()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="#Broker.purge_queue"><code class="docutils literal notranslate"><span class="pre">Broker.purge_queue()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="#Broker.delete_queue"><code class="docutils literal notranslate"><span class="pre">Broker.delete_queue()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="#Broker.queue_size"><code class="docutils literal notranslate"><span class="pre">Broker.queue_size()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="#Broker.lock_size"><code class="docutils literal notranslate"><span class="pre">Broker.lock_size()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="#Broker.ping"><code class="docutils literal notranslate"><span class="pre">Broker.ping()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="#Broker.info"><code class="docutils literal notranslate"><span class="pre">Broker.info()</span></code></a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="#brokers.get_broker"><code class="docutils literal notranslate"><span class="pre">brokers.get_broker()</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="tasks.html"> Tasks</a></li>
<li class="toctree-l1"><a class="reference internal" href="group.html"> Groups</a></li>
<li class="toctree-l1"><a class="reference internal" href="iterable.html"> Iterable</a></li>
<li class="toctree-l1"><a class="reference internal" href="chain.html"> Chains</a></li>
<li class="toctree-l1"><a class="reference internal" href="schedules.html"> Schedules</a></li>
<li class="toctree-l1"><a class="reference internal" href="cluster.html"> Cluster</a></li>
<li class="toctree-l1"><a class="reference internal" href="monitor.html"> Monitor</a></li>
<li class="toctree-l1"><a class="reference internal" href="admin.html"> Admin</a></li>
<li class="toctree-l1"><a class="reference internal" href="errors.html"> Errors</a></li>
<li class="toctree-l1"><a class="reference internal" href="signals.html"> Signals</a></li>
<li class="toctree-l1"><a class="reference internal" href="architecture.html"> Architecture</a></li>
<li class="toctree-l1"><a class="reference internal" href="examples.html"> Examples</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Django Q2</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="Page navigation">
<ul class="wy-breadcrumbs">
<li><a href="index.html" class="icon icon-home" aria-label="Home"></a></li>
<li class="breadcrumb-item active">Brokers</li>
<li class="wy-breadcrumbs-aside">
<a href="_sources/brokers.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<section id="brokers">
<h1>Brokers<a class="headerlink" href="#brokers" title="Link to this heading"></a></h1>
<p>The broker sits between your Django instances and your Django Q2 cluster instances; accepting, saving and delivering task packages.
Currently we support a variety of brokers.</p>
<p>The default Redis broker does not support message receipts.
This means that in case of a catastrophic failure of the cluster server or worker timeouts, tasks that were being executed get lost.
Keep in mind this is not the same as a failing task. If a tasks code crashes, this should only lead to a failed task status.</p>
<p>Even though this might be acceptable in some use cases, you might prefer brokers with message receipts support.
These guarantee delivery by waiting for the cluster to send a receipt after the task has been processed.
In case a receipt has not been received after a set time, the task package is put back in the queue.
Django Q2 supports this behavior by setting the <a class="reference internal" href="configure.html#retry"><span class="std std-ref">retry</span></a> timer on brokers that support message receipts.</p>
<p>Some pointers:</p>
<ul class="simple">
<li><p>Don’t set the <a class="reference internal" href="configure.html#retry"><span class="std std-ref">retry</span></a> timer to a lower or equal number than the task timeout.</p></li>
<li><p>Retry time includes time the task spends waiting in the clusters internal queue.</p></li>
<li><p>Don’t set the <a class="reference internal" href="configure.html#queue-limit"><span class="std std-ref">queue_limit</span></a> so high that tasks time out while waiting to be processed.</p></li>
<li><p>In case a task is worked on twice, the task result will be updated with the latest results.</p></li>
<li><p>In some rare cases a non-atomic broker will re-queue a task after it has been acknowledged.</p></li>
<li><p>If a task runs twice and a previous run has succeeded, the new result will be discarded.</p></li>
<li><p>Limiting the number of retries is handled globally in your actual broker’s settings.</p></li>
</ul>
<p>Support for more brokers is being worked on.</p>
<section id="redis">
<h2>Redis<a class="headerlink" href="#redis" title="Link to this heading"></a></h2>
<p>The default broker for Django Q2 clusters.</p>
<ul class="simple">
<li><p>Atomic</p></li>
<li><p>Requires <a class="reference external" href="https://github.com/andymccurdy/redis-py">Redis-py</a> client library: <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">redis</span></code></p></li>
<li><p>Does not need cache framework for monitoring</p></li>
<li><p>Does not support receipts</p></li>
<li><p>Can use existing <a class="reference internal" href="configure.html#django-redis"><span class="std std-ref">django_redis</span></a> connections.</p></li>
<li><p>Configure with <a class="reference internal" href="configure.html#redis-configuration"><span class="std std-ref">redis</span></a>-py compatible configuration</p></li>
</ul>
</section>
<section id="ironmq">
<h2>IronMQ<a class="headerlink" href="#ironmq" title="Link to this heading"></a></h2>
<p>This HTTP based queue service is both available directly via <a class="reference external" href="http://www.iron.io/mq/">Iron.io</a> and as an add-on on Heroku.</p>
<ul class="simple">
<li><p>Delivery receipts</p></li>
<li><p>Supports bulk dequeue</p></li>
<li><p>Needs Django’s <a class="reference external" href="https://docs.djangoproject.com/en/4.0/topics/cache/#setting-up-the-cache">Cache framework</a> configured for monitoring</p></li>
<li><p>Requires the <a class="reference external" href="https://github.com/iron-io/iron_mq_python">iron-mq</a> client library: <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">iron-mq</span></code></p></li>
<li><p>See the <a class="reference internal" href="configure.html#ironmq-configuration"><span class="std std-ref">iron_mq</span></a> configuration section for options.</p></li>
</ul>
</section>
<section id="amazon-sqs">
<h2>Amazon SQS<a class="headerlink" href="#amazon-sqs" title="Link to this heading"></a></h2>
<p>Amazon’s Simple Queue Service is another HTTP based message queue.
Although <a class="reference external" href="https://aws.amazon.com/sqs/">SQS</a> is not the fastest, it is stable, cheap and convenient if you already use AWS.</p>
<ul class="simple">
<li><p>Delivery receipts</p></li>
<li><p>Maximum message size is 256Kb</p></li>
<li><p>Supports bulk dequeue up to 10 messages with a maximum total size of 256Kb</p></li>
<li><p>Needs Django’s <a class="reference external" href="https://docs.djangoproject.com/en/4.0/topics/cache/#setting-up-the-cache">Cache framework</a> configured for monitoring</p></li>
<li><p>Requires the <a class="reference external" href="https://github.com/boto/boto3">boto3</a> client library: <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">boto3</span></code></p></li>
<li><p>See the <a class="reference internal" href="configure.html#sqs-configuration"><span class="std std-ref">sqs</span></a> configuration section for options.</p></li>
</ul>
</section>
<section id="mongodb">
<h2>MongoDB<a class="headerlink" href="#mongodb" title="Link to this heading"></a></h2>
<p>This highly scalable NoSQL database makes for a very fast and reliably persistent at-least-once message broker.
Usually available on most PaaS providers, as <a class="reference external" href="https://www.mongodb.org/">open-source</a> or commercial <a class="reference external" href="https://www.mongodb.com/lp/download/mongodb-enterprise">enterprise</a> edition.</p>
<ul class="simple">
<li><p>Delivery receipts</p></li>
<li><p>Needs Django’s <a class="reference external" href="https://docs.djangoproject.com/en/4.0/topics/cache/#setting-up-the-cache">Cache framework</a> configured for monitoring</p></li>
<li><p>Can be configured as the Django cache-backend through several open-source cache providers.</p></li>
<li><p>Requires the <a class="reference external" href="https://github.com/mongodb/mongo-python-driver">pymongo</a> driver: <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">pymongo</span></code></p></li>
<li><p>See the <a class="reference internal" href="configure.html#mongo-configuration"><span class="std std-ref">mongo</span></a> configuration section for options.</p></li>
</ul>
</section>
<section id="django-orm">
<span id="orm-broker"></span><h2>Django ORM<a class="headerlink" href="#django-orm" title="Link to this heading"></a></h2>
<p>Select this to use Django’s database backend as a message broker.
Unless you have configured a dedicated database backend for it, this should probably not be your first choice for a high traffic setup.
However for a medium message rate and scheduled tasks, this is the most convenient guaranteed delivery broker.</p>
<ul class="simple">
<li><p>Delivery receipts</p></li>
<li><p>Supports bulk dequeue</p></li>
<li><p>Needs Django’s <a class="reference external" href="https://docs.djangoproject.com/en/4.0/topics/cache/#setting-up-the-cache">Cache framework</a> configured for monitoring</p></li>
<li><p>Can be <a class="reference external" href="https://docs.djangoproject.com/en/4.0/topics/cache/#database-caching">configured</a> as its own cache backend.</p></li>
<li><p>Queue editable in Django Admin</p></li>
<li><p>See the <a class="reference internal" href="configure.html#orm-configuration"><span class="std std-ref">orm</span></a> configuration on how to set it up.</p></li>
</ul>
</section>
<section id="custom-broker">
<h2>Custom Broker<a class="headerlink" href="#custom-broker" title="Link to this heading"></a></h2>
<p>You can override the <a class="reference internal" href="#Broker" title="Broker"><code class="xref py py-class docutils literal notranslate"><span class="pre">Broker</span></code></a> or any of its existing derived broker types.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># example Custom broker.py</span>
<span class="kn">from</span> <span class="nn">django_q.brokers</span> <span class="kn">import</span> <span class="n">Broker</span>
<span class="k">class</span> <span class="nc">CustomBroker</span><span class="p">(</span><span class="n">Broker</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s1">'My Custom Broker'</span>
</pre></div>
</div>
<p>Using the <a class="reference internal" href="configure.html#broker-class"><span class="std std-ref">broker_class</span></a> configuration setting you can then instruct Django Q2 to use this instead of one of the existing brokers:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># example Custom broker class connection</span>
<span class="n">Q_CLUSTER</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'name'</span><span class="p">:</span> <span class="s1">'Custom'</span><span class="p">,</span>
<span class="s1">'workers'</span><span class="p">:</span> <span class="mi">8</span><span class="p">,</span>
<span class="s1">'timeout'</span><span class="p">:</span> <span class="mi">60</span><span class="p">,</span>
<span class="s1">'broker_class: '</span><span class="n">myapp</span><span class="o">.</span><span class="n">broker</span><span class="o">.</span><span class="n">CustomBroker</span><span class="s1">'</span>
<span class="p">}</span>
</pre></div>
</div>
<p>If you do write a custom broker for one of the many message queueing servers out there we don’t support yet, please consider contributing it to the project.</p>
</section>
<section id="reference">
<h2>Reference<a class="headerlink" href="#reference" title="Link to this heading"></a></h2>
<p>The <a class="reference internal" href="#Broker" title="Broker"><code class="xref py py-class docutils literal notranslate"><span class="pre">Broker</span></code></a> class is used internally to communicate with the different types of brokers.
You can override this class if you want to contribute and support your own broker.</p>
<dl class="py class">
<dt class="sig sig-object py" id="Broker">
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">Broker</span></span><a class="headerlink" href="#Broker" title="Link to this definition"></a></dt>
<dd><dl class="py method">
<dt class="sig sig-object py" id="Broker.async_task">
<span class="sig-name descname"><span class="pre">async_task</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">task</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#Broker.async_task" title="Link to this definition"></a></dt>
<dd><p>Sends a task package to the broker queue and returns a tracking id if available.</p>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="Broker.dequeue">
<span class="sig-name descname"><span class="pre">dequeue</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#Broker.dequeue" title="Link to this definition"></a></dt>
<dd><p>Gets packages from the broker and returns a list of tuples with a tracking id and the package.</p>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="Broker.acknowledge">
<span class="sig-name descname"><span class="pre">acknowledge</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">id</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#Broker.acknowledge" title="Link to this definition"></a></dt>
<dd><p>Notifies the broker that the task has been processed.
Only works with brokers that support delivery receipts.</p>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="Broker.fail">
<span class="sig-name descname"><span class="pre">fail</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">id</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#Broker.fail" title="Link to this definition"></a></dt>
<dd><p>Tells the broker that the message failed to be processed by the cluster.
Only available on brokers that support this.
Currently only occurs when a cluster fails to unpack a task package.</p>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="Broker.delete">
<span class="sig-name descname"><span class="pre">delete</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">id</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#Broker.delete" title="Link to this definition"></a></dt>
<dd><p>Instructs the broker to delete this message from the queue.</p>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="Broker.purge_queue">
<span class="sig-name descname"><span class="pre">purge_queue</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#Broker.purge_queue" title="Link to this definition"></a></dt>
<dd><p>Empties the current queue of all messages.</p>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="Broker.delete_queue">
<span class="sig-name descname"><span class="pre">delete_queue</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#Broker.delete_queue" title="Link to this definition"></a></dt>
<dd><p>Deletes the current queue from the broker.</p>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="Broker.queue_size">
<span class="sig-name descname"><span class="pre">queue_size</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#Broker.queue_size" title="Link to this definition"></a></dt>
<dd><p>Returns the amount of messages in the brokers queue.</p>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="Broker.lock_size">
<span class="sig-name descname"><span class="pre">lock_size</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#Broker.lock_size" title="Link to this definition"></a></dt>
<dd><p>Optional method that returns the number of messages currently awaiting acknowledgement.
Only implemented on brokers that support it.</p>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="Broker.ping">
<span class="sig-name descname"><span class="pre">ping</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#Broker.ping" title="Link to this definition"></a></dt>
<dd><p>Returns True if the broker can be reached.</p>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="Broker.info">
<span class="sig-name descname"><span class="pre">info</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#Broker.info" title="Link to this definition"></a></dt>
<dd><p>Shows the name and version of the currently configured broker.</p>
</dd></dl>
</dd></dl>
<dl class="py function">
<dt class="sig sig-object py" id="brokers.get_broker">
<span class="sig-prename descclassname"><span class="pre">brokers.</span></span><span class="sig-name descname"><span class="pre">get_broker</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#brokers.get_broker" title="Link to this definition"></a></dt>
<dd><p>Returns a <a class="reference internal" href="#Broker" title="Broker"><code class="xref py py-class docutils literal notranslate"><span class="pre">Broker</span></code></a> instance based on the current configuration.</p>
</dd></dl>
</section>
</section>
</div>
</div>
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
<a href="configure.html" class="btn btn-neutral float-left" title="Configuration" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
<a href="tasks.html" class="btn btn-neutral float-right" title="Tasks" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
</div>
<hr/>
<div role="contentinfo">
<p>© Copyright 2015-2023, Ilan Steemers - 2022, Stan Triepels.</p>
</div>
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script>
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>