File: //usr/share/doc/python3-django-q/html/configure.html
<!DOCTYPE html>
<html class="writer-html5" lang="en" data-content_root="./">
<head>
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Configuration — Django Q2 1.6.1 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b76e3c8a" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=86f27845" />
<link rel="shortcut icon" href="_static/favicon.ico"/>
<script src="_static/jquery.js?v=8dae8fb0"></script>
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
<script src="_static/documentation_options.js?v=d6a008b6"></script>
<script src="_static/doctools.js?v=888ff710"></script>
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
<script src="_static/js/theme.js"></script>
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="Brokers" href="brokers.html" />
<link rel="prev" title="Installation" href="install.html" />
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="index.html" class="icon icon-home">
Django Q2
</a>
<div class="version">
1.6
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="install.html"> Installation</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#"> Configuration</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#name">name</a></li>
<li class="toctree-l2"><a class="reference internal" href="#workers">workers</a></li>
<li class="toctree-l2"><a class="reference internal" href="#daemonize-workers">daemonize_workers</a></li>
<li class="toctree-l2"><a class="reference internal" href="#recycle">recycle</a></li>
<li class="toctree-l2"><a class="reference internal" href="#max-rss">max_rss</a></li>
<li class="toctree-l2"><a class="reference internal" href="#timeout">timeout</a></li>
<li class="toctree-l2"><a class="reference internal" href="#time-zone">time_zone</a></li>
<li class="toctree-l2"><a class="reference internal" href="#ack-failures">ack_failures</a></li>
<li class="toctree-l2"><a class="reference internal" href="#max-attempts">max_attempts</a></li>
<li class="toctree-l2"><a class="reference internal" href="#retry">retry</a></li>
<li class="toctree-l2"><a class="reference internal" href="#compress">compress</a></li>
<li class="toctree-l2"><a class="reference internal" href="#save-limit">save_limit</a></li>
<li class="toctree-l2"><a class="reference internal" href="#save-limit-per">save_limit_per</a></li>
<li class="toctree-l2"><a class="reference internal" href="#guard-cycle">guard_cycle</a></li>
<li class="toctree-l2"><a class="reference internal" href="#sync">sync</a></li>
<li class="toctree-l2"><a class="reference internal" href="#queue-limit">queue_limit</a></li>
<li class="toctree-l2"><a class="reference internal" href="#label">label</a></li>
<li class="toctree-l2"><a class="reference internal" href="#catch-up">catch_up</a></li>
<li class="toctree-l2"><a class="reference internal" href="#redis">redis</a></li>
<li class="toctree-l2"><a class="reference internal" href="#django-redis">django_redis</a></li>
<li class="toctree-l2"><a class="reference internal" href="#iron-mq">iron_mq</a></li>
<li class="toctree-l2"><a class="reference internal" href="#sqs">sqs</a></li>
<li class="toctree-l2"><a class="reference internal" href="#orm">orm</a></li>
<li class="toctree-l2"><a class="reference internal" href="#mongo">mongo</a></li>
<li class="toctree-l2"><a class="reference internal" href="#mongo-db">mongo_db</a></li>
<li class="toctree-l2"><a class="reference internal" href="#broker-class">broker_class</a></li>
<li class="toctree-l2"><a class="reference internal" href="#bulk">bulk</a></li>
<li class="toctree-l2"><a class="reference internal" href="#poll">poll</a></li>
<li class="toctree-l2"><a class="reference internal" href="#cache">cache</a></li>
<li class="toctree-l2"><a class="reference internal" href="#cached">cached</a></li>
<li class="toctree-l2"><a class="reference internal" href="#scheduler">scheduler</a></li>
<li class="toctree-l2"><a class="reference internal" href="#error-reporter">error_reporter</a></li>
<li class="toctree-l2"><a class="reference internal" href="#cpu-affinity">cpu_affinity</a></li>
<li class="toctree-l2"><a class="reference internal" href="#alt-clusters">ALT_CLUSTERS</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="brokers.html"> Brokers</a></li>
<li class="toctree-l1"><a class="reference internal" href="tasks.html"> Tasks</a></li>
<li class="toctree-l1"><a class="reference internal" href="group.html"> Groups</a></li>
<li class="toctree-l1"><a class="reference internal" href="iterable.html"> Iterable</a></li>
<li class="toctree-l1"><a class="reference internal" href="chain.html"> Chains</a></li>
<li class="toctree-l1"><a class="reference internal" href="schedules.html"> Schedules</a></li>
<li class="toctree-l1"><a class="reference internal" href="cluster.html"> Cluster</a></li>
<li class="toctree-l1"><a class="reference internal" href="monitor.html"> Monitor</a></li>
<li class="toctree-l1"><a class="reference internal" href="admin.html"> Admin</a></li>
<li class="toctree-l1"><a class="reference internal" href="errors.html"> Errors</a></li>
<li class="toctree-l1"><a class="reference internal" href="signals.html"> Signals</a></li>
<li class="toctree-l1"><a class="reference internal" href="architecture.html"> Architecture</a></li>
<li class="toctree-l1"><a class="reference internal" href="examples.html"> Examples</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Django Q2</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="Page navigation">
<ul class="wy-breadcrumbs">
<li><a href="index.html" class="icon icon-home" aria-label="Home"></a></li>
<li class="breadcrumb-item active">Configuration</li>
<li class="wy-breadcrumbs-aside">
<a href="_sources/configure.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<section id="configuration">
<h1>Configuration<a class="headerlink" href="#configuration" title="Link to this heading"></a></h1>
<p>Configuration is handled via the <code class="docutils literal notranslate"><span class="pre">Q_CLUSTER</span></code> dictionary in your <code class="file docutils literal notranslate"><span class="pre">settings.py</span></code></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># settings.py example</span>
<span class="n">Q_CLUSTER</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'name'</span><span class="p">:</span> <span class="s1">'myproject'</span><span class="p">,</span>
<span class="s1">'workers'</span><span class="p">:</span> <span class="mi">8</span><span class="p">,</span>
<span class="s1">'recycle'</span><span class="p">:</span> <span class="mi">500</span><span class="p">,</span>
<span class="s1">'timeout'</span><span class="p">:</span> <span class="mi">60</span><span class="p">,</span>
<span class="s1">'compress'</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s1">'save_limit'</span><span class="p">:</span> <span class="mi">250</span><span class="p">,</span>
<span class="s1">'queue_limit'</span><span class="p">:</span> <span class="mi">500</span><span class="p">,</span>
<span class="s1">'cpu_affinity'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="s1">'label'</span><span class="p">:</span> <span class="s1">'Django Q2'</span><span class="p">,</span>
<span class="s1">'redis'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'host'</span><span class="p">:</span> <span class="s1">'127.0.0.1'</span><span class="p">,</span>
<span class="s1">'port'</span><span class="p">:</span> <span class="mi">6379</span><span class="p">,</span>
<span class="s1">'db'</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> <span class="p">},</span>
<span class="s1">'ALT_CLUSTERS'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'long'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'timeout'</span><span class="p">:</span> <span class="mi">3000</span><span class="p">,</span>
<span class="s1">'retry'</span><span class="p">:</span> <span class="mi">3600</span><span class="p">,</span>
<span class="s1">'max_attempts'</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span>
<span class="p">},</span>
<span class="s1">'short'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'timeout'</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span>
<span class="s1">'max_attempts'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="p">},</span>
<span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>All configuration settings are optional:</p>
<section id="name">
<span id="id1"></span><h2>name<a class="headerlink" href="#name" title="Link to this heading"></a></h2>
<p>Used to differentiate between projects using the same broker.
On most broker types this will be used as the queue name.
Defaults to <code class="docutils literal notranslate"><span class="pre">'default'</span></code>.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Tasks are signed. When a worker encounters a task with an invalid signature, it will be discarded or failed.</p>
</div>
</section>
<section id="workers">
<h2>workers<a class="headerlink" href="#workers" title="Link to this heading"></a></h2>
<p>The number of workers to use in the cluster. Defaults to CPU count of the current host, but can be set to a custom number. <a class="footnote-reference brackets" href="#f1" id="id2" role="doc-noteref"><span class="fn-bracket">[</span>1<span class="fn-bracket">]</span></a></p>
</section>
<section id="daemonize-workers">
<h2>daemonize_workers<a class="headerlink" href="#daemonize-workers" title="Link to this heading"></a></h2>
<p>Set the daemon flag when spawning workers. You may need to disable this flag if your worker needs to spawn child process but be careful with orphaned child processes in case of sudden termination of the main process.
Defaults to <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p>
</section>
<section id="recycle">
<h2>recycle<a class="headerlink" href="#recycle" title="Link to this heading"></a></h2>
<p>The number of tasks a worker will process before recycling . Useful to release memory resources on a regular basis. Defaults to <code class="docutils literal notranslate"><span class="pre">500</span></code>.</p>
</section>
<section id="max-rss">
<h2>max_rss<a class="headerlink" href="#max-rss" title="Link to this heading"></a></h2>
<p>The maximum resident set size in kilobytes before a worker will recycle and release resources. Useful for limiting memory usage.
Only supported on platforms that implement the python resource module or install the <a class="reference internal" href="install.html#psutil-package"><span class="std std-ref">psutil</span></a> module.
Defaults to <code class="docutils literal notranslate"><span class="pre">None</span></code>.</p>
</section>
<section id="timeout">
<span id="id3"></span><h2>timeout<a class="headerlink" href="#timeout" title="Link to this heading"></a></h2>
<p>The number of seconds a worker is allowed to spend on a task before it’s terminated. Defaults to <code class="docutils literal notranslate"><span class="pre">None</span></code>, meaning it will never time out.
Set this to something that makes sense for your project. Can be overridden for individual tasks.</p>
<p>See <a class="reference internal" href="#retry"><span class="std std-ref">retry</span></a> for details how to set values for timeout and retry.</p>
</section>
<section id="time-zone">
<span id="id4"></span><h2>time_zone<a class="headerlink" href="#time-zone" title="Link to this heading"></a></h2>
<p>The timezone that is used for task scheduling. Use this if you are having issue with DST. The scheduler uses UTC to calculate the next date and will therefore ignore any DST changes. This will cause 1 hour or 0.5 hour changes in the schedule when time is moved one hour ahead or back. Defaults to <cite>settings.TIME_ZONE</cite> if <cite>USE_TZ</cite> is enabled.</p>
</section>
<section id="ack-failures">
<span id="id5"></span><h2>ack_failures<a class="headerlink" href="#ack-failures" title="Link to this heading"></a></h2>
<p>When set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, also acknowledge unsuccessful tasks. This causes failed tasks to be considered as successful deliveries, thereby removing them from the task queue. Can also be set per-task by passing the <code class="docutils literal notranslate"><span class="pre">ack_failure</span></code> option to <a class="reference internal" href="tasks.html#django_q.async_task" title="django_q.async_task"><code class="xref py py-func docutils literal notranslate"><span class="pre">async_task()</span></code></a>. Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p>
</section>
<section id="max-attempts">
<span id="id6"></span><h2>max_attempts<a class="headerlink" href="#max-attempts" title="Link to this heading"></a></h2>
<p>Limit the number of retry attempts for failed tasks. Set to 0 for infinite retries. Defaults to 0</p>
</section>
<section id="retry">
<span id="id7"></span><h2>retry<a class="headerlink" href="#retry" title="Link to this heading"></a></h2>
<p>The number of seconds a broker will wait for a cluster to finish a task, before it’s presented again.
Only works with brokers that support delivery receipts. Defaults to 60.</p>
<p>The value must be bigger than the time it takes to complete longest task, i.e. <a class="reference internal" href="#timeout"><span class="std std-ref">timeout</span></a> must be less than retry value and all tasks must complete
in less time than the selected retry time. If this does not hold, i.e. the retry value is less than timeout or less than it takes to finish a task,
Django-Q2 will start the task again if the used broker supports receipts.</p>
<p>For example, with the following code</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># settings.py</span>
<span class="n">Q_CLUSTER</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'retry'</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span>
<span class="s1">'workers'</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span>
<span class="s1">'orm'</span><span class="p">:</span> <span class="s1">'default'</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># example.py</span>
<span class="kn">from</span> <span class="nn">django_q.tasks</span> <span class="kn">import</span> <span class="n">async_task</span>
<span class="n">async_task</span><span class="p">(</span><span class="s1">'time.sleep'</span><span class="p">,</span> <span class="mi">22</span><span class="p">)</span>
</pre></div>
</div>
<p>First, <code class="docutils literal notranslate"><span class="pre">time.sleep</span></code> is called by the first worker. After 5 seconds second worker will also call <code class="docutils literal notranslate"><span class="pre">time.sleep</span></code> because retry time has exceeded and the
broker return the task again for the cluster. After 21 seconds from the call to <code class="docutils literal notranslate"><span class="pre">async_task</span></code> all four workers are running the <code class="docutils literal notranslate"><span class="pre">time.sleep(22)</span></code> call
and there is one retry in queue; tasks are started after 0, 5, 10, 15 and 20 seconds after the <code class="docutils literal notranslate"><span class="pre">async_task</span></code> was called. After 22 seconds the first
worker completes and the task is acknowledged in the broker and the task is not added to task queue anymore but the task that was already in the run queue
will run also. So in this example, <code class="docutils literal notranslate"><span class="pre">time.sleep</span></code> was called 5 times.</p>
<p>Note also that the above issue might cause all workers to run the same long running task preventing new tasks from starting shortly after the task has been
started by <code class="docutils literal notranslate"><span class="pre">async_task</span></code>. In this case the retry time handling could cause the task that has not been started by any worker to be put on work queue again
(even multiple times).</p>
</section>
<section id="compress">
<h2>compress<a class="headerlink" href="#compress" title="Link to this heading"></a></h2>
<p>Compresses task packages to the broker. Useful for large payloads, but can add overhead when used with many small packages.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code></p>
</section>
<section id="save-limit">
<span id="id8"></span><h2>save_limit<a class="headerlink" href="#save-limit" title="Link to this heading"></a></h2>
<dl class="simple">
<dt>Limits the amount of successful tasks saved to Django.</dt><dd><ul class="simple">
<li><p>Set to <code class="docutils literal notranslate"><span class="pre">0</span></code> for unlimited.</p></li>
<li><p>Set to <code class="docutils literal notranslate"><span class="pre">-1</span></code> for no success storage at all.</p></li>
<li><p>Defaults to <code class="docutils literal notranslate"><span class="pre">250</span></code></p></li>
<li><p>Failures are always saved.</p></li>
</ul>
</dd>
</dl>
</section>
<section id="save-limit-per">
<h2>save_limit_per<a class="headerlink" href="#save-limit-per" title="Link to this heading"></a></h2>
<dl class="simple">
<dt>The above <code class="docutils literal notranslate"><span class="pre">save_limit</span></code> for successful tasks can be fine tuned per task type using</dt><dd><ul class="simple">
<li><p>Set to <code class="docutils literal notranslate"><span class="pre">"group"</span></code> to store the tasks per group</p></li>
<li><p>Other possible values are <code class="docutils literal notranslate"><span class="pre">"func"</span></code>, <code class="docutils literal notranslate"><span class="pre">"name"</span></code>, <code class="docutils literal notranslate"><span class="pre">None</span></code></p></li>
<li><p>Defaults to <code class="docutils literal notranslate"><span class="pre">None</span></code></p></li>
</ul>
</dd>
</dl>
</section>
<section id="guard-cycle">
<h2>guard_cycle<a class="headerlink" href="#guard-cycle" title="Link to this heading"></a></h2>
<p>Guard loop sleep in seconds, must be greater than 0 and less than 60.</p>
</section>
<section id="sync">
<span id="id9"></span><h2>sync<a class="headerlink" href="#sync" title="Link to this heading"></a></h2>
<p>When set to <code class="docutils literal notranslate"><span class="pre">True</span></code> this configuration option forces all <a class="reference internal" href="tasks.html#django_q.async_task" title="django_q.async_task"><code class="xref py py-func docutils literal notranslate"><span class="pre">async_task()</span></code></a> calls to be run with <code class="docutils literal notranslate"><span class="pre">sync=True</span></code>.
Effectively making everything synchronous. Useful for testing. Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p>
</section>
<section id="queue-limit">
<span id="id10"></span><h2>queue_limit<a class="headerlink" href="#queue-limit" title="Link to this heading"></a></h2>
<p>This does not limit the amount of tasks that can be queued on the broker, but rather how many tasks are kept in memory by a single cluster.
Setting this to a reasonable number, can help balance the workload and the memory overhead of each individual cluster.
Defaults to <code class="docutils literal notranslate"><span class="pre">workers**2</span></code>.</p>
</section>
<section id="label">
<h2>label<a class="headerlink" href="#label" title="Link to this heading"></a></h2>
<p>The label used for the Django Admin page. Defaults to <code class="docutils literal notranslate"><span class="pre">'Django</span> <span class="pre">Q2'</span></code></p>
</section>
<section id="catch-up">
<span id="id11"></span><h2>catch_up<a class="headerlink" href="#catch-up" title="Link to this heading"></a></h2>
<p>The default behavior for schedules that didn’t run while a cluster was down, is to play catch up and execute all the missed time slots until things are back on schedule.
You can override this behavior by setting <code class="docutils literal notranslate"><span class="pre">catch_up</span></code> to <code class="docutils literal notranslate"><span class="pre">False</span></code>. This will make those schedules run only once when the cluster starts and normal scheduling resumes.
Defaults to <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p>
</section>
<section id="redis">
<span id="redis-configuration"></span><h2>redis<a class="headerlink" href="#redis" title="Link to this heading"></a></h2>
<p>Connection settings for Redis. Defaults:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># redis defaults</span>
<span class="n">Q_CLUSTER</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'redis'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'host'</span><span class="p">:</span> <span class="s1">'localhost'</span><span class="p">,</span>
<span class="s1">'port'</span><span class="p">:</span> <span class="mi">6379</span><span class="p">,</span>
<span class="s1">'db'</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
<span class="s1">'password'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s1">'socket_timeout'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s1">'charset'</span><span class="p">:</span> <span class="s1">'utf-8'</span><span class="p">,</span>
<span class="s1">'errors'</span><span class="p">:</span> <span class="s1">'strict'</span><span class="p">,</span>
<span class="s1">'unix_socket_path'</span><span class="p">:</span> <span class="kc">None</span>
<span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>It’s also possible to use a Redis connection URI:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Q_CLUSTER</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'redis'</span><span class="p">:</span> <span class="s1">'redis://h:asdfqwer1234asdf@ec2-111-1-1-1.compute-1.amazonaws.com:111'</span>
<span class="p">}</span>
</pre></div>
</div>
<p>For more information on these settings please refer to the <a class="reference external" href="https://github.com/andymccurdy/redis-py">Redis-py</a> documentation</p>
</section>
<section id="django-redis">
<span id="id12"></span><h2>django_redis<a class="headerlink" href="#django-redis" title="Link to this heading"></a></h2>
<p>If you are already using <a class="reference external" href="https://github.com/niwinz/django-redis">django-redis</a> for your caching, you can take advantage of its excellent connection backend by supplying the name
of the cache connection you want to use instead of a direct Redis connection:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># example django-redis connection</span>
<span class="n">Q_CLUSTER</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'name'</span><span class="p">:</span> <span class="s1">'DJRedis'</span><span class="p">,</span>
<span class="s1">'workers'</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span>
<span class="s1">'timeout'</span><span class="p">:</span> <span class="mi">90</span><span class="p">,</span>
<span class="s1">'django_redis'</span><span class="p">:</span> <span class="s1">'default'</span>
<span class="p">}</span>
</pre></div>
</div>
<div class="admonition tip">
<p class="admonition-title">Tip</p>
<p>Django Q2 uses your <code class="docutils literal notranslate"><span class="pre">SECRET_KEY</span></code> to sign task packages and prevent task crossover. So make sure you have it set up in your Django settings.</p>
</div>
</section>
<section id="iron-mq">
<span id="ironmq-configuration"></span><h2>iron_mq<a class="headerlink" href="#iron-mq" title="Link to this heading"></a></h2>
<p>Connection settings for IronMQ:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># example IronMQ connection</span>
<span class="n">Q_CLUSTER</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'name'</span><span class="p">:</span> <span class="s1">'IronBroker'</span><span class="p">,</span>
<span class="s1">'workers'</span><span class="p">:</span> <span class="mi">8</span><span class="p">,</span>
<span class="s1">'timeout'</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span>
<span class="s1">'retry'</span><span class="p">:</span> <span class="mi">60</span><span class="p">,</span>
<span class="s1">'queue_limit'</span><span class="p">:</span> <span class="mi">50</span><span class="p">,</span>
<span class="s1">'bulk'</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span>
<span class="s1">'iron_mq'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'host'</span><span class="p">:</span> <span class="s1">'mq-aws-us-east-1.iron.io'</span><span class="p">,</span>
<span class="s1">'token'</span><span class="p">:</span> <span class="s1">'Et1En7.....0LuW39Q'</span><span class="p">,</span>
<span class="s1">'project_id'</span><span class="p">:</span> <span class="s1">'500f7b....b0f302e9'</span>
<span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>All connection keywords are supported. See the <a class="reference external" href="https://github.com/iron-io/iron_mq_python#configure">iron-mq</a> library for more info</p>
</section>
<section id="sqs">
<span id="sqs-configuration"></span><h2>sqs<a class="headerlink" href="#sqs" title="Link to this heading"></a></h2>
<p>To use Amazon SQS as a broker you need to provide the AWS region and credentials either via the config, or any other boto3 configuration method:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># example SQS broker connection</span>
<span class="n">Q_CLUSTER</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'name'</span><span class="p">:</span> <span class="s1">'SQSExample'</span><span class="p">,</span>
<span class="s1">'workers'</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span>
<span class="s1">'timeout'</span><span class="p">:</span> <span class="mi">60</span><span class="p">,</span>
<span class="s1">'retry'</span><span class="p">:</span> <span class="mi">90</span><span class="p">,</span>
<span class="s1">'queue_limit'</span><span class="p">:</span> <span class="mi">100</span><span class="p">,</span>
<span class="s1">'bulk'</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span>
<span class="s1">'sqs'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'aws_region'</span><span class="p">:</span> <span class="s1">'us-east-1'</span><span class="p">,</span> <span class="c1"># optional</span>
<span class="s1">'aws_access_key_id'</span><span class="p">:</span> <span class="s1">'ac-Idr.....YwflZBaaxI'</span><span class="p">,</span> <span class="c1"># optional</span>
<span class="s1">'aws_secret_access_key'</span><span class="p">:</span> <span class="s1">'500f7b....b0f302e9'</span> <span class="c1"># optional</span>
<span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Please make sure these credentials have proper SQS access.</p>
<p>Amazon SQS only supports a bulk setting between 1 and 10, with the total payload not exceeding 256kb.</p>
</section>
<section id="orm">
<span id="orm-configuration"></span><h2>orm<a class="headerlink" href="#orm" title="Link to this heading"></a></h2>
<p>If you want to use Django’s database backend as a message broker, set the <code class="docutils literal notranslate"><span class="pre">orm</span></code> keyword to the database connection you want it to use:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># example ORM broker connection</span>
<span class="n">Q_CLUSTER</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'name'</span><span class="p">:</span> <span class="s1">'DjangORM'</span><span class="p">,</span>
<span class="s1">'workers'</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span>
<span class="s1">'timeout'</span><span class="p">:</span> <span class="mi">90</span><span class="p">,</span>
<span class="s1">'retry'</span><span class="p">:</span> <span class="mi">120</span><span class="p">,</span>
<span class="s1">'queue_limit'</span><span class="p">:</span> <span class="mi">50</span><span class="p">,</span>
<span class="s1">'bulk'</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span>
<span class="s1">'orm'</span><span class="p">:</span> <span class="s1">'default'</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Using the Django ORM backend will also enable the Queued Tasks table in the Admin.</p>
<p>If you need better performance , you should consider using a different database backend than the main project.
Set <code class="docutils literal notranslate"><span class="pre">orm</span></code> to the name of that database connection and make sure you run migrations on it using the <code class="docutils literal notranslate"><span class="pre">--database</span></code> option.</p>
</section>
<section id="mongo">
<span id="mongo-configuration"></span><h2>mongo<a class="headerlink" href="#mongo" title="Link to this heading"></a></h2>
<p>To use MongoDB as a message broker you simply provide the connection information in a dictionary:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># example MongoDB broker connection</span>
<span class="n">Q_CLUSTER</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'name'</span><span class="p">:</span> <span class="s1">'MongoDB'</span><span class="p">,</span>
<span class="s1">'workers'</span><span class="p">:</span> <span class="mi">8</span><span class="p">,</span>
<span class="s1">'timeout'</span><span class="p">:</span> <span class="mi">60</span><span class="p">,</span>
<span class="s1">'retry'</span><span class="p">:</span> <span class="mi">70</span><span class="p">,</span>
<span class="s1">'queue_limit'</span><span class="p">:</span> <span class="mi">100</span><span class="p">,</span>
<span class="s1">'mongo'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'host'</span><span class="p">:</span> <span class="s1">'127.0.0.1'</span><span class="p">,</span>
<span class="s1">'port'</span><span class="p">:</span> <span class="mi">27017</span>
<span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">mongo</span></code> dictionary can contain any of the parameters exposed by pymongo’s <a class="reference external" href="https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient">MongoClient</a>
If you want to use a mongodb uri, you can supply it as the <code class="docutils literal notranslate"><span class="pre">host</span></code> parameter.</p>
</section>
<section id="mongo-db">
<h2>mongo_db<a class="headerlink" href="#mongo-db" title="Link to this heading"></a></h2>
<p>When using the MongoDB broker you can optionally provide a database name to use for the queues.
Defaults to default database if available, otherwise <code class="docutils literal notranslate"><span class="pre">django-q</span></code></p>
</section>
<section id="broker-class">
<span id="id13"></span><h2>broker_class<a class="headerlink" href="#broker-class" title="Link to this heading"></a></h2>
<p>You can use a custom broker class for your cluster workers:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># example Custom broker class connection</span>
<span class="n">Q_CLUSTER</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'name'</span><span class="p">:</span> <span class="s1">'Custom'</span><span class="p">,</span>
<span class="s1">'workers'</span><span class="p">:</span> <span class="mi">8</span><span class="p">,</span>
<span class="s1">'timeout'</span><span class="p">:</span> <span class="mi">60</span><span class="p">,</span>
<span class="s1">'broker_class'</span><span class="p">:</span> <span class="s1">'myapp.broker.CustomBroker'</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Make sure your <code class="docutils literal notranslate"><span class="pre">CustomBroker</span></code> class inherits from either the base <a class="reference internal" href="brokers.html#Broker" title="Broker"><code class="xref py py-class docutils literal notranslate"><span class="pre">Broker</span></code></a> class or one of its children.</p>
</section>
<section id="bulk">
<span id="id14"></span><h2>bulk<a class="headerlink" href="#bulk" title="Link to this heading"></a></h2>
<p>Sets the number of messages each cluster tries to get from the broker per call. Setting this on supported brokers can improve performance.
Especially HTTP based or very high latency servers can benefit from bulk dequeue.
Keep in mind however that settings this too high can degrade performance with multiple clusters or very large task packages.</p>
<p>Not supported by the default Redis broker.
Defaults to <code class="docutils literal notranslate"><span class="pre">1</span></code>.</p>
</section>
<section id="poll">
<h2>poll<a class="headerlink" href="#poll" title="Link to this heading"></a></h2>
<p>Sets the queue polling interval for database brokers that don’t have a blocking call. Currently only affects the ORM and MongoDB brokers.
Defaults to <code class="docutils literal notranslate"><span class="pre">0.2</span></code> (seconds).</p>
</section>
<section id="cache">
<h2>cache<a class="headerlink" href="#cache" title="Link to this heading"></a></h2>
<p>For some brokers, you will need to set up the Django <a class="reference external" href="https://docs.djangoproject.com/en/2.2/topics/cache/#setting-up-the-cache">cache framework</a>
to gather statistics for the monitor. You can indicate which cache to use by setting this value. Defaults to <code class="docutils literal notranslate"><span class="pre">default</span></code>.</p>
</section>
<section id="cached">
<span id="id15"></span><h2>cached<a class="headerlink" href="#cached" title="Link to this heading"></a></h2>
<p>Switches all task and result functions from using the database backend to the cache backend. This is the same as setting the keyword <code class="docutils literal notranslate"><span class="pre">cached=True</span></code> on all task functions.
Instead of a bool this can also be set to the number of seconds you want the cache to retain results. e.g. <code class="docutils literal notranslate"><span class="pre">cached=60</span></code></p>
</section>
<section id="scheduler">
<h2>scheduler<a class="headerlink" href="#scheduler" title="Link to this heading"></a></h2>
<p>You can disable the scheduler by setting this option to <code class="docutils literal notranslate"><span class="pre">False</span></code>. This will reduce a little overhead if you’re not using schedules, but is most useful if you want to temporarily disable all schedules.
Defaults to <code class="docutils literal notranslate"><span class="pre">True</span></code></p>
</section>
<section id="error-reporter">
<span id="id16"></span><h2>error_reporter<a class="headerlink" href="#error-reporter" title="Link to this heading"></a></h2>
<p>You can redirect worker exceptions directly to various error reporters (for example <a class="reference external" href="https://rollbar.com/">Rollbar</a> or <a class="reference external" href="https://docs.sentry.io/">Sentry</a>) by installing Django Q2 with the necessary <a class="reference external" href="https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies">extras</a>.</p>
<p>To enable installed error reporters, you must provide the configuration settings required by an error reporter extension:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># error_reporter config--rollbar example</span>
<span class="n">Q_CLUSTER</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'error_reporter'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'rollbar'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'access_token'</span><span class="p">:</span> <span class="s1">'32we33a92a5224jiww8982'</span><span class="p">,</span>
<span class="s1">'environment'</span><span class="p">:</span> <span class="s1">'Django-Q2'</span>
<span class="p">}</span>
<span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>For more information on error reporters and developing error reporting plugins for Django Q2, see <a class="reference internal" href="errors.html"><span class="doc">errors</span></a>.</p>
</section>
<section id="cpu-affinity">
<h2>cpu_affinity<a class="headerlink" href="#cpu-affinity" title="Link to this heading"></a></h2>
<p>Sets the number of processor each worker can use. This does not affect auxiliary processes like the sentinel or monitor and is only useful for tweaking the performance of very high traffic clusters.
The affinity number has to be higher than zero and less than the total number of processors to have any effect. Defaults to using all processors:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># processor affinity example.</span>
<span class="mi">4</span> <span class="n">processors</span><span class="p">,</span> <span class="mi">4</span> <span class="n">workers</span><span class="p">,</span> <span class="n">cpu_affinity</span><span class="p">:</span> <span class="mi">1</span>
<span class="n">worker</span> <span class="mi">1</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">2</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">3</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">2</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">4</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">3</span><span class="p">]</span>
<span class="mi">4</span> <span class="n">processors</span><span class="p">,</span> <span class="mi">4</span> <span class="n">workers</span><span class="p">,</span> <span class="n">cpu_affinity</span><span class="p">:</span> <span class="mi">2</span>
<span class="n">worker</span> <span class="mi">1</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">2</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">3</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">4</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
<span class="mi">8</span> <span class="n">processors</span><span class="p">,</span> <span class="mi">8</span> <span class="n">workers</span><span class="p">,</span> <span class="n">cpu_affinity</span><span class="p">:</span> <span class="mi">3</span>
<span class="n">worker</span> <span class="mi">1</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">2</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">3</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">6</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">0</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">4</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">5</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">6</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">7</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]</span>
<span class="n">worker</span> <span class="mi">8</span> <span class="n">cpu</span> <span class="p">[</span><span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">7</span><span class="p">]</span>
</pre></div>
</div>
<p>In some cases, setting the cpu affinity for your workers can lead to performance improvements, especially if the load is high and consists of many repeating small tasks.
Start with an affinity of 1 and work your way up. You will have to experiment with what works best for you.
As a rule of thumb; cpu_affinity 1 favors repetitive short running tasks, while no affinity benefits longer running tasks.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The <code class="docutils literal notranslate"><span class="pre">cpu_affinity</span></code> setting requires the optional <a class="reference internal" href="install.html#psutil-package"><span class="std std-ref">psutil</span></a> module.</p>
<p><em>Psutil does not support cpu affinity on OS X at this time.</em></p>
</div>
</section>
<section id="alt-clusters">
<span id="id17"></span><h2>ALT_CLUSTERS<a class="headerlink" href="#alt-clusters" title="Link to this heading"></a></h2>
<p>For multiple clusters working on multiple queues to run in one Django site.
ALT_CLUSTERS should be a dict with cluster_name as its key, and the value is the configuration for the cluster
with the key as its name. The configuration items are consistent with Q_CLUSTER,
except for a few items such as name/cluster_name/ALT_CLUSTER, which are not available of course.</p>
<p>See <span class="xref std std-ref">multiple-queues</span>.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>For a cluster, if its name is in ALT_CLUSTERS, the config item in ALT_CLUSTER will override
the same config item in the Q_CLUSTER root. Other config items in Q_CLUSTER root remain in effect for this cluster.</p>
</div>
<p class="rubric" id="module-django_q">Footnotes</p>
<aside class="footnote-list brackets">
<aside class="footnote brackets" id="f1" role="doc-footnote">
<span class="label"><span class="fn-bracket">[</span><a role="doc-backlink" href="#id2">1</a><span class="fn-bracket">]</span></span>
<p>Uses <code class="xref py py-func docutils literal notranslate"><span class="pre">multiprocessing.cpu_count()</span></code> which can fail on some platforms. If so , please set the worker count in the configuration manually or install <a class="reference internal" href="install.html#psutil-package"><span class="std std-ref">psutil</span></a> to provide an alternative cpu count method.</p>
</aside>
</aside>
</section>
</section>
</div>
</div>
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
<a href="install.html" class="btn btn-neutral float-left" title="Installation" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
<a href="brokers.html" class="btn btn-neutral float-right" title="Brokers" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
</div>
<hr/>
<div role="contentinfo">
<p>© Copyright 2015-2023, Ilan Steemers - 2022, Stan Triepels.</p>
</div>
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script>
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>